Skip to main content

turbovault_plugin_vector/
lib.rs

1//! `vector_search`: hybrid dense + lexical semantic search over the active
2//! vault, as a compiled-in TurboVault plugin.
3//!
4//! Wraps [`turbovault_vector::IndexEngine`] behind the plugin boundary:
5//! notes are read through [`VaultApi`], the index persists under this
6//! plugin's [`PluginStorage`] namespace, and it is kept current by a
7//! background worker that reconciles on every [`HookBus`] event (with an
8//! initial pass in [`PluginProvider::start`] so a note edited while the
9//! process was down is not missed) plus a resync every tool call, matching
10//! how the host's own derived state stays fresh. Local tool names are
11//! advertised namespaced as `vector_search_*`.
12//!
13//! See the crate README for what is and is not implemented yet (notably:
14//! one engine per process, tied to whichever vault is active when it is
15//! first built).
16
17use std::collections::HashMap;
18use std::sync::Arc;
19
20use async_trait::async_trait;
21use serde_json::{Value, json};
22use tokio::sync::{Mutex, OnceCell};
23use turbomcp_types::ToolInputSchema;
24use turbovault_plugin_api::{
25    HookBus, HookRecvError, NoteListing, Plugin, PluginContext, PluginDescriptor, PluginError,
26    PluginProvider, PluginRequestContext, PluginResult, PluginStorage, ShutdownSignal, Tool,
27    ToolResult, VaultApi, VaultDescriptor,
28};
29use turbovault_vector::{
30    EmbeddingEngine, IndexEngine, Model2VecEmbedder, NoteRecord, VectorConfig, VectorError,
31};
32
33/// Key holding a JSON-encoded, partial [`VectorConfig`] override. Missing
34/// fields fall back to the built-in defaults; the file itself is optional.
35const CONFIG_KEY: &str = "config.json";
36/// Key holding the reconcile cursor: `HashMap<path, NoteListing>` as last
37/// observed, so a restart does not treat every note as newly changed.
38const MANIFEST_KEY: &str = "manifest.json";
39/// Key holding the chunk-id allocator's high-water mark.
40const META_KEY: &str = "meta.json";
41/// Directory under which each note's [`NoteRecord`] is stored, one key per
42/// note so an edit to one note does not rewrite every other note's vectors.
43///
44/// No trailing slash: `PluginStorage::list`'s prefix is validated as a key
45/// (via `validate_key`) unless it is empty, and a trailing `/` produces an
46/// empty final segment, which `validate_key` rejects. A plain string prefix
47/// still matches every `notes/...` key, since `list` does a `starts_with`
48/// match rather than directory-aware matching.
49const NOTES_DIR: &str = "notes";
50
51fn note_key(path: &str) -> String {
52    format!("{NOTES_DIR}/{path}.json")
53}
54
55fn verr(error: VectorError) -> PluginError {
56    match error {
57        // A config problem (e.g. no model_path set) is the caller's to fix,
58        // not a server failure.
59        VectorError::Config(message) => PluginError::invalid_input(message),
60        VectorError::Embedding(message)
61        | VectorError::Index(message)
62        | VectorError::Snapshot(message) => PluginError::internal(message),
63    }
64}
65
66/// Builds the embedder an [`EngineHandle`] embeds with, from the resolved
67/// [`VectorConfig`]. A trait rather than a bare function pointer so a test
68/// factory can close over state (a call counter, say) if it needs to.
69type EmbedderFactory =
70    Arc<dyn Fn(&VectorConfig) -> PluginResult<Arc<dyn EmbeddingEngine>> + Send + Sync>;
71
72fn model2vec_embedder_factory(config: &VectorConfig) -> PluginResult<Arc<dyn EmbeddingEngine>> {
73    Ok(Arc::new(
74        Model2VecEmbedder::load(&config.model_path).map_err(verr)?,
75    ))
76}
77
78/// Compiled-in factory for the `vector_search` plugin.
79pub struct VectorSearchPlugin {
80    embedder_factory: EmbedderFactory,
81}
82
83impl VectorSearchPlugin {
84    /// The production plugin: embeds with a real, locally-loaded Model2Vec
85    /// model (see [`turbovault_vector::Model2VecEmbedder`]).
86    pub fn new() -> Self {
87        Self {
88            embedder_factory: Arc::new(model2vec_embedder_factory),
89        }
90    }
91
92    /// Build with a caller-supplied embedder factory instead of loading a
93    /// real model. For tests: this is how the plugin boundary (tool
94    /// namespacing, storage, the change feed) gets exercised end to end
95    /// without a model file to hand and without any network access. See
96    /// `turbovault_vector::embedding::testing` for a ready-made fake.
97    pub fn with_embedder_factory(
98        factory: impl Fn(&VectorConfig) -> PluginResult<Arc<dyn EmbeddingEngine>>
99        + Send
100        + Sync
101        + 'static,
102    ) -> Self {
103        Self {
104            embedder_factory: Arc::new(factory),
105        }
106    }
107}
108
109impl Default for VectorSearchPlugin {
110    fn default() -> Self {
111        Self::new()
112    }
113}
114
115impl Plugin for VectorSearchPlugin {
116    fn descriptor(&self) -> PluginDescriptor {
117        PluginDescriptor::new(
118            "vector_search",
119            "Vector Search",
120            env!("CARGO_PKG_VERSION"),
121            "Hybrid dense + lexical semantic search over the active vault",
122        )
123    }
124
125    fn build(&self, context: PluginContext) -> PluginResult<Arc<dyn PluginProvider>> {
126        let handle = Arc::new(EngineHandle {
127            vault: context.vault,
128            storage: context.storage,
129            embedder_factory: Arc::clone(&self.embedder_factory),
130            engine: OnceCell::new(),
131            seen: Mutex::new(HashMap::new()),
132        });
133        Ok(Arc::new(VectorSearchProvider {
134            handle,
135            hooks: context.hooks,
136            shutdown: context.shutdown,
137            worker: Mutex::new(None),
138        }))
139    }
140}
141
142/// Lazily-built engine plus the storage and reconciliation state around it.
143///
144/// One engine per process. Built lazily (loading a model is not free, and
145/// `Plugin::build` is synchronous) against whichever vault is active the
146/// first time anything asks for it; [`Self::engine_for`] fails loudly rather
147/// than silently searching the wrong vault's index if the active vault later
148/// changes. See the crate README for the reasoning and the multi-vault
149/// follow-up this leaves open.
150struct EngineHandle {
151    vault: VaultApi,
152    storage: PluginStorage,
153    embedder_factory: EmbedderFactory,
154    engine: OnceCell<(String, IndexEngine)>,
155    /// Reconcile cursor: what this plugin last saw for each note, so
156    /// [`Self::reconcile`] only re-reads notes whose listing moved.
157    seen: Mutex<HashMap<String, NoteListing>>,
158}
159
160impl EngineHandle {
161    /// The engine, built and restored from persisted state on first use.
162    /// Returns an error if the active vault has changed since the engine was
163    /// built, rather than silently answering from the wrong vault's index.
164    async fn engine_for(&self, active: &VaultDescriptor) -> PluginResult<&IndexEngine> {
165        let (built_for, engine) = self
166            .engine
167            .get_or_try_init(|| async {
168                let config = self.load_config(&active.name).await?;
169                let embedder = (self.embedder_factory)(&config)?;
170                let index_engine = IndexEngine::new(embedder, config);
171                self.restore(&active.name, &index_engine).await?;
172                Ok::<(String, IndexEngine), PluginError>((active.name.clone(), index_engine))
173            })
174            .await?;
175        if built_for != &active.name {
176            return Err(PluginError::unavailable(format!(
177                "vector_search built its index against vault {built_for:?} and does not yet \
178                 support switching to {:?} without a server restart",
179                active.name
180            )));
181        }
182        Ok(engine)
183    }
184
185    /// Convenience for a call site that has not already resolved the active
186    /// vault.
187    async fn engine(&self) -> PluginResult<&IndexEngine> {
188        let active = self.vault.active_vault().await?;
189        self.engine_for(&active).await
190    }
191
192    async fn load_config(&self, vault: &str) -> PluginResult<VectorConfig> {
193        match self.storage.get(vault, CONFIG_KEY).await? {
194            Some(bytes) => serde_json::from_slice(&bytes)
195                .map_err(|error| PluginError::internal(format!("invalid {CONFIG_KEY}: {error}"))),
196            None => Ok(VectorConfig::default()),
197        }
198    }
199
200    /// Load every persisted note record plus the chunk-id counter and the
201    /// reconcile cursor, and rebuild `index_engine`'s in-memory state from
202    /// them. Calls the embedder for nothing.
203    async fn restore(&self, vault: &str, index_engine: &IndexEngine) -> PluginResult<()> {
204        let keys = self.storage.list(vault, NOTES_DIR).await?;
205        let mut notes = Vec::with_capacity(keys.len());
206        for key in &keys {
207            let Some(bytes) = self.storage.get(vault, key).await? else {
208                continue;
209            };
210            match serde_json::from_slice::<NoteRecord>(&bytes) {
211                Ok(note) => notes.push(note),
212                Err(error) => {
213                    tracing::warn!("vector_search: dropping corrupt snapshot {key:?}: {error}");
214                }
215            }
216        }
217
218        let next_chunk_id = match self.storage.get(vault, META_KEY).await? {
219            Some(bytes) => serde_json::from_slice::<turbovault_vector::IndexMeta>(&bytes)
220                .map(|meta| meta.next_chunk_id)
221                .unwrap_or(1),
222            None => 1,
223        };
224        index_engine
225            .restore(notes, next_chunk_id)
226            .await
227            .map_err(verr)?;
228
229        if let Some(bytes) = self.storage.get(vault, MANIFEST_KEY).await?
230            && let Ok(manifest) = serde_json::from_slice::<HashMap<String, NoteListing>>(&bytes)
231        {
232            *self.seen.lock().await = manifest;
233        }
234        Ok(())
235    }
236
237    /// Persist one note's current record (or delete its key, if the engine
238    /// no longer has it) and, if anything changed, the manifest and counter
239    /// alongside it. Written in that order deliberately: if the process dies
240    /// between the note write and the manifest write, the note is simply
241    /// re-checked (and, via its own content hash, a no-op) on the next
242    /// reconcile. The other order could permanently skip a real change.
243    async fn persist_note(
244        &self,
245        vault: &str,
246        path: &str,
247        index_engine: &IndexEngine,
248    ) -> PluginResult<()> {
249        match index_engine.snapshot_note(path).await {
250            Some(record) => {
251                let bytes = serde_json::to_vec(&record)
252                    .map_err(|error| PluginError::internal(error.to_string()))?;
253                self.storage.put(vault, &note_key(path), &bytes).await?;
254            }
255            None => self.storage.delete(vault, &note_key(path)).await?,
256        }
257        Ok(())
258    }
259
260    async fn persist_manifest(&self, vault: &str) -> PluginResult<()> {
261        let seen = self.seen.lock().await;
262        let bytes =
263            serde_json::to_vec(&*seen).map_err(|error| PluginError::internal(error.to_string()))?;
264        drop(seen);
265        self.storage.put(vault, MANIFEST_KEY, &bytes).await
266    }
267
268    async fn persist_meta(&self, vault: &str, index_engine: &IndexEngine) -> PluginResult<()> {
269        let meta = turbovault_vector::IndexMeta {
270            model: index_engine.model_name().to_string(),
271            dims: index_engine.dims(),
272            next_chunk_id: index_engine.next_chunk_id(),
273        };
274        let bytes =
275            serde_json::to_vec(&meta).map_err(|error| PluginError::internal(error.to_string()))?;
276        self.storage.put(vault, META_KEY, &bytes).await
277    }
278
279    /// Re-read every note whose `list_notes_detailed` listing moved since
280    /// last seen, remove notes that vanished, and persist what changed.
281    /// `IndexEngine::update_note` skips unchanged content at chunk
282    /// granularity, so a listing that looked changed but was not (a touch,
283    /// a re-save with no edits) costs one read and no re-embedding.
284    ///
285    /// Returns the number of notes that were actually re-indexed or removed.
286    async fn reconcile(&self) -> PluginResult<usize> {
287        let active = self.vault.active_vault().await?;
288        let index_engine = self.engine_for(&active).await?;
289        let current = self.vault.list_notes_detailed(&active.name).await?;
290        let current: HashMap<String, NoteListing> = current
291            .into_iter()
292            .map(|listing| (listing.path.clone(), listing))
293            .collect();
294
295        let seen = self.seen.lock().await;
296        let to_check: Vec<String> = current
297            .iter()
298            .filter(|(path, listing)| {
299                !seen
300                    .get(*path)
301                    .is_some_and(|previous| listing.looks_unchanged_from(previous))
302            })
303            .map(|(path, _)| path.clone())
304            .collect();
305        let gone: Vec<String> = seen
306            .keys()
307            .filter(|path| !current.contains_key(*path))
308            .cloned()
309            .collect();
310        drop(seen);
311
312        if to_check.is_empty() && gone.is_empty() {
313            return Ok(0);
314        }
315
316        let mut changed = 0usize;
317        if !to_check.is_empty() {
318            for snapshot in self.vault.read_notes(&active.name, &to_check).await? {
319                let did_change = index_engine
320                    .update_note(&snapshot.path, &snapshot.content)
321                    .await
322                    .map_err(verr)?;
323                if did_change {
324                    changed += 1;
325                    self.persist_note(&active.name, &snapshot.path, index_engine)
326                        .await?;
327                }
328                if let Some(listing) = current.get(&snapshot.path) {
329                    self.seen
330                        .lock()
331                        .await
332                        .insert(snapshot.path.clone(), listing.clone());
333                }
334            }
335        }
336        for path in &gone {
337            if index_engine.remove_note(path).await {
338                changed += 1;
339                self.storage.delete(&active.name, &note_key(path)).await?;
340            }
341            self.seen.lock().await.remove(path);
342        }
343
344        if changed > 0 || !gone.is_empty() {
345            self.persist_manifest(&active.name).await?;
346            self.persist_meta(&active.name, index_engine).await?;
347        }
348        Ok(changed)
349    }
350
351    /// Drop the engine's in-memory state and every persisted note record and
352    /// cursor, so the next reconcile treats the whole vault as unseen and
353    /// re-embeds everything. For a config or model change, or recovering
354    /// from a snapshot a caller no longer trusts.
355    async fn hard_reset(&self) -> PluginResult<usize> {
356        let active = self.vault.active_vault().await?;
357        let index_engine = self.engine_for(&active).await?;
358        index_engine.clear().await.map_err(verr)?;
359
360        let stale_keys = self.storage.list(&active.name, NOTES_DIR).await?;
361        for key in &stale_keys {
362            self.storage.delete(&active.name, key).await?;
363        }
364        self.storage.delete(&active.name, META_KEY).await?;
365        self.seen.lock().await.clear();
366        self.persist_manifest(&active.name).await?;
367
368        self.reconcile().await
369    }
370}
371
372pub struct VectorSearchProvider {
373    handle: Arc<EngineHandle>,
374    hooks: HookBus,
375    shutdown: ShutdownSignal,
376    worker: Mutex<Option<tokio::task::JoinHandle<()>>>,
377}
378
379impl VectorSearchProvider {
380    async fn search(&self, arguments: &Value) -> PluginResult<ToolResult> {
381        let query = arguments
382            .get("query")
383            .and_then(Value::as_str)
384            .map(str::trim)
385            .filter(|query| !query.is_empty())
386            .ok_or_else(|| PluginError::invalid_input("query is required and must not be empty"))?;
387        let limit = arguments
388            .get("limit")
389            .and_then(Value::as_u64)
390            .map(|limit| limit.clamp(1, 100) as usize)
391            .unwrap_or(10);
392        let hybrid = arguments
393            .get("hybrid")
394            .and_then(Value::as_bool)
395            .unwrap_or(true);
396
397        // Reconcile before serving, the same contract the host's own derived
398        // reads follow: a search answers from state at least as fresh as the
399        // last debounced pass, not from whatever the background worker
400        // happened to have applied already.
401        self.handle.reconcile().await?;
402        let index_engine = self.handle.engine().await?;
403        let results = index_engine
404            .search(query, limit, hybrid)
405            .await
406            .map_err(verr)?;
407        ok_json(json!({
408            "query": query,
409            "hybrid": hybrid,
410            "count": results.len(),
411            "results": results,
412        }))
413    }
414
415    async fn reindex(&self) -> PluginResult<ToolResult> {
416        let reindexed = self.handle.hard_reset().await?;
417        ok_json(json!({ "reindexed_notes": reindexed }))
418    }
419
420    async fn status(&self) -> PluginResult<ToolResult> {
421        if let Some((_, index_engine)) = self.handle.engine.get() {
422            let stats = index_engine.stats().await;
423            let config = index_engine.config();
424            return ok_json(json!({
425                "configured": true,
426                "model": stats.model,
427                "dims": stats.dims,
428                "indexed_notes": stats.indexed_notes,
429                "indexed_chunks": stats.indexed_chunks,
430                "chunk_max_chars": config.chunk_max_chars,
431                "chunk_overlap_chars": config.chunk_overlap_chars,
432                "lexical_weight": config.lexical_weight,
433                "rrf_k": config.rrf_k,
434            }));
435        }
436
437        // Not built yet: report the resolved config (including any override
438        // in `config.json`) without forcing a model load, so `status` stays
439        // cheap and safe to call before vector search is configured at all.
440        let active = self.handle.vault.active_vault().await?;
441        let config = self.handle.load_config(&active.name).await?;
442        ok_json(json!({
443            "configured": config.is_configured(),
444            "model_path": config.model_path,
445            "chunk_max_chars": config.chunk_max_chars,
446            "chunk_overlap_chars": config.chunk_overlap_chars,
447            "lexical_weight": config.lexical_weight,
448            "rrf_k": config.rrf_k,
449            "indexed_notes": 0,
450            "indexed_chunks": 0,
451            "note": if config.is_configured() {
452                "model not loaded yet; call vector_search_search or vector_search_reindex to load it"
453            } else {
454                "not configured; set model_path in .turbovault/plugins/vector_search/config.json \
455                 to a local Model2Vec model directory"
456            },
457        }))
458    }
459}
460
461#[async_trait]
462impl PluginProvider for VectorSearchProvider {
463    fn tools(&self) -> Vec<Tool> {
464        vec![
465            tool(
466                "search",
467                "Hybrid semantic search over the vault; returns the most relevant note chunks",
468                json!({
469                    "type": "object",
470                    "properties": {
471                        "query": { "type": "string", "description": "Natural-language query" },
472                        "limit": { "type": "integer", "minimum": 1, "maximum": 100, "description": "Max results (default 10)" },
473                        "hybrid": { "type": "boolean", "description": "Fuse BM25 lexical ranking in via Reciprocal Rank Fusion (default true); false for dense-only" }
474                    },
475                    "required": ["query"]
476                }),
477            ),
478            tool(
479                "reindex",
480                "Force a full re-embed of the vault (normally indexing is incremental, driven by edits)",
481                json!({ "type": "object", "properties": {} }),
482            ),
483            tool(
484                "status",
485                "Report index status: model, dimensions, indexed note/chunk counts, resolved config",
486                json!({ "type": "object", "properties": {} }),
487            ),
488        ]
489    }
490
491    async fn call_tool(
492        &self,
493        name: &str,
494        arguments: Value,
495        _context: PluginRequestContext,
496    ) -> PluginResult<ToolResult> {
497        match name {
498            "search" => self.search(&arguments).await,
499            "reindex" => self.reindex().await,
500            "status" => self.status().await,
501            other => Err(PluginError::not_found(format!("unknown tool {other:?}"))),
502        }
503    }
504
505    async fn start(&self) -> PluginResult<()> {
506        // Prime the index before entering the event loop: a note edited
507        // while the process was down should be searchable on the first
508        // query, not only after the first live event arrives.
509        if let Err(error) = self.handle.reconcile().await {
510            tracing::warn!("vector_search: initial reconcile failed: {error}");
511        }
512
513        let handle = Arc::clone(&self.handle);
514        let shutdown = self.shutdown.clone();
515        let mut events = self
516            .hooks
517            .subscribe()
518            .map_err(|error| PluginError::unavailable(error.to_string()))?;
519
520        let task = tokio::spawn(async move {
521            loop {
522                tokio::select! {
523                    _ = shutdown.cancelled() => break,
524                    event = events.recv() => match event {
525                        // Every event, of whatever kind, is answered the same
526                        // way: reconcile from listings. That is what makes
527                        // this correct for `ResyncRequired` too, with no
528                        // special case for it.
529                        Ok(_envelope) => {
530                            if let Err(error) = handle.reconcile().await {
531                                tracing::warn!("vector_search: reconcile failed: {error}");
532                            }
533                        }
534                        Err(HookRecvError::Lagged { .. }) => {
535                            if let Err(error) = handle.reconcile().await {
536                                tracing::warn!("vector_search: reconcile after lag failed: {error}");
537                            }
538                        }
539                        Err(HookRecvError::Closed) => break,
540                        Err(HookRecvError::Empty) => continue,
541                    },
542                }
543            }
544        });
545        *self.worker.lock().await = Some(task);
546        Ok(())
547    }
548
549    async fn shutdown(&self) {
550        if let Some(task) = self.worker.lock().await.take() {
551            let _ = task.await;
552        }
553    }
554}
555
556fn tool(name: &str, description: &str, schema: Value) -> Tool {
557    Tool::new(name, description).with_schema(ToolInputSchema::from_value(schema))
558}
559
560fn ok_json(value: Value) -> PluginResult<ToolResult> {
561    ToolResult::json(&value).map_err(|error| PluginError::internal(error.to_string()))
562}
563
564#[cfg(test)]
565mod tests {
566    use super::*;
567    use std::sync::Mutex as StdMutex;
568    use std::sync::atomic::{AtomicU64, Ordering};
569
570    use turbovault_plugin_api::{
571        HookBus, NoteSnapshot, PluginIdentity, PluginStore, ShutdownTrigger, VaultHost,
572        WriteNoteRequest, WriteReceipt,
573    };
574    use turbovault_vector::embedding::testing::{CountingEmbedder, DeterministicEmbedder};
575
576    const VAULT: &str = "test-vault";
577    const DIMS: usize = 32;
578
579    /// In-memory [`PluginStore`], namespaced by `(vault, key)` exactly like
580    /// the production `FilePluginStore`, so a test can share one across two
581    /// `EngineHandle`s to simulate a restart.
582    #[derive(Default)]
583    struct FakeStore {
584        data: StdMutex<HashMap<(String, String), Vec<u8>>>,
585    }
586
587    #[async_trait]
588    impl PluginStore for FakeStore {
589        async fn get(&self, vault: &str, key: &str) -> PluginResult<Option<Vec<u8>>> {
590            Ok(self
591                .data
592                .lock()
593                .unwrap()
594                .get(&(vault.to_string(), key.to_string()))
595                .cloned())
596        }
597
598        async fn put(&self, vault: &str, key: &str, value: &[u8]) -> PluginResult<()> {
599            self.data
600                .lock()
601                .unwrap()
602                .insert((vault.to_string(), key.to_string()), value.to_vec());
603            Ok(())
604        }
605
606        async fn delete(&self, vault: &str, key: &str) -> PluginResult<()> {
607            self.data
608                .lock()
609                .unwrap()
610                .remove(&(vault.to_string(), key.to_string()));
611            Ok(())
612        }
613
614        async fn list(&self, vault: &str, prefix: &str) -> PluginResult<Vec<String>> {
615            let data = self.data.lock().unwrap();
616            let mut keys: Vec<String> = data
617                .keys()
618                .filter(|(entry_vault, key)| entry_vault == vault && key.starts_with(prefix))
619                .map(|(_, key)| key.clone())
620                .collect();
621            keys.sort();
622            Ok(keys)
623        }
624    }
625
626    /// In-memory [`VaultHost`], with an explicit "clock" so a test can
627    /// control exactly what `list_notes_detailed` reports without racing a
628    /// real filesystem's timestamp resolution.
629    #[derive(Default)]
630    struct FakeVault {
631        notes: StdMutex<HashMap<String, (String, u64)>>, // path -> (content, observed_at)
632        clock: AtomicU64,
633    }
634
635    impl FakeVault {
636        fn put_note(&self, path: &str, content: &str) {
637            let observed_at = self.clock.fetch_add(1, Ordering::SeqCst);
638            self.notes
639                .lock()
640                .unwrap()
641                .insert(path.to_string(), (content.to_string(), observed_at));
642        }
643
644        fn remove_note(&self, path: &str) {
645            self.notes.lock().unwrap().remove(path);
646        }
647    }
648
649    #[async_trait]
650    impl VaultHost for FakeVault {
651        async fn active_vault(&self) -> PluginResult<VaultDescriptor> {
652            Ok(VaultDescriptor::new(VAULT, "direct"))
653        }
654
655        async fn list_notes(&self, _vault: &str) -> PluginResult<Vec<String>> {
656            Ok(self.notes.lock().unwrap().keys().cloned().collect())
657        }
658
659        async fn list_notes_detailed(&self, _vault: &str) -> PluginResult<Vec<NoteListing>> {
660            Ok(self
661                .notes
662                .lock()
663                .unwrap()
664                .iter()
665                .map(|(path, (content, observed_at))| {
666                    NoteListing::new(path.clone(), content.len() as u64, Some(*observed_at))
667                })
668                .collect())
669        }
670
671        async fn read_note(&self, _vault: &str, path: &str) -> PluginResult<NoteSnapshot> {
672            let notes = self.notes.lock().unwrap();
673            let (content, _) = notes
674                .get(path)
675                .ok_or_else(|| PluginError::not_found(path.to_string()))?;
676            Ok(NoteSnapshot::new(VAULT, path, content.clone(), "v1"))
677        }
678
679        async fn read_notes(
680            &self,
681            _vault: &str,
682            paths: &[String],
683        ) -> PluginResult<Vec<NoteSnapshot>> {
684            let notes = self.notes.lock().unwrap();
685            Ok(paths
686                .iter()
687                .filter_map(|path| {
688                    notes
689                        .get(path)
690                        .map(|(content, _)| NoteSnapshot::new(VAULT, path, content.clone(), "v1"))
691                })
692                .collect())
693        }
694
695        async fn write_note(&self, _request: WriteNoteRequest) -> PluginResult<WriteReceipt> {
696            Err(PluginError::unavailable("FakeVault is read-only"))
697        }
698
699        async fn read_config(
700            &self,
701            _vault: &str,
702            _relative_path: &str,
703        ) -> PluginResult<Option<Vec<u8>>> {
704            Ok(None)
705        }
706    }
707
708    fn vault_api(host: Arc<FakeVault>) -> VaultApi {
709        let identity = PluginIdentity::new("vector_search", Default::default()).expect("identity");
710        VaultApi::new(host, identity)
711    }
712
713    fn handle_with(
714        store: Arc<FakeStore>,
715        vault: Arc<FakeVault>,
716        embedder_calls: Arc<CountingEmbedder>,
717    ) -> EngineHandle {
718        EngineHandle {
719            vault: vault_api(vault),
720            storage: PluginStorage::new(store),
721            embedder_factory: Arc::new(move |_config: &VectorConfig| {
722                Ok(Arc::clone(&embedder_calls) as Arc<dyn EmbeddingEngine>)
723            }),
724            engine: OnceCell::new(),
725            seen: Mutex::new(HashMap::new()),
726        }
727    }
728
729    #[tokio::test]
730    async fn reconcile_indexes_new_notes_and_persists_them() {
731        let store = Arc::new(FakeStore::default());
732        let vault = Arc::new(FakeVault::default());
733        vault.put_note("alpha.md", "Alpha unique text.");
734        vault.put_note("beta.md", "Beta unique text.");
735        let embedder = Arc::new(CountingEmbedder::new(DIMS));
736        let handle = handle_with(Arc::clone(&store), vault, embedder);
737
738        let changed = handle.reconcile().await.expect("reconcile");
739        assert_eq!(changed, 2);
740
741        let keys = store.list(VAULT, NOTES_DIR).await.expect("list");
742        assert_eq!(keys, vec![note_key("alpha.md"), note_key("beta.md")]);
743        assert!(store.get(VAULT, MANIFEST_KEY).await.expect("get").is_some());
744        assert!(store.get(VAULT, META_KEY).await.expect("get").is_some());
745
746        // A second reconcile with nothing changed must not touch either
747        // index or re-persist anything.
748        let changed_again = handle.reconcile().await.expect("second reconcile");
749        assert_eq!(changed_again, 0);
750    }
751
752    #[tokio::test]
753    async fn restart_restores_the_index_without_re_embedding() {
754        // The fake embedder hashes whole inputs into a bag of features with
755        // no notion of "contains as a substring" (unlike a real model), so a
756        // query has to match the indexed text itself to land above
757        // min_similarity on the dense-only path this test exercises.
758        const NOTE_CONTENT: &str = "Some searchable unique content.";
759        let store = Arc::new(FakeStore::default());
760        let vault = Arc::new(FakeVault::default());
761        vault.put_note("note.md", NOTE_CONTENT);
762
763        let embedder = Arc::new(CountingEmbedder::new(DIMS));
764        let handle = handle_with(
765            Arc::clone(&store),
766            Arc::clone(&vault),
767            Arc::clone(&embedder),
768        );
769        handle.reconcile().await.expect("initial reconcile");
770        let embedded_before_restart = embedder.count();
771        assert!(embedded_before_restart > 0);
772
773        // A fresh handle sharing the same store stands in for a process
774        // restart: everything on disk survives, nothing in memory does.
775        let restarted_embedder = Arc::new(CountingEmbedder::new(DIMS));
776        let restarted = handle_with(store, vault, Arc::clone(&restarted_embedder));
777        let active = restarted.vault.active_vault().await.expect("active vault");
778        let index_engine = restarted.engine_for(&active).await.expect("engine");
779        let stats = index_engine.stats().await;
780        assert_eq!(stats.indexed_notes, 1);
781        assert_eq!(stats.indexed_chunks, 1);
782        assert_eq!(
783            restarted_embedder.count(),
784            0,
785            "restore must not call the embedder"
786        );
787
788        let hits = index_engine
789            .search(NOTE_CONTENT, 5, false)
790            .await
791            .expect("search");
792        assert_eq!(hits[0].path, "note.md");
793    }
794
795    #[tokio::test]
796    async fn removed_notes_are_dropped_from_index_and_storage() {
797        let store = Arc::new(FakeStore::default());
798        let vault = Arc::new(FakeVault::default());
799        vault.put_note("note.md", "Content to be removed.");
800        let embedder = Arc::new(CountingEmbedder::new(DIMS));
801        let handle = handle_with(Arc::clone(&store), Arc::clone(&vault), embedder);
802        handle.reconcile().await.expect("index it first");
803        assert!(
804            store
805                .get(VAULT, &note_key("note.md"))
806                .await
807                .unwrap()
808                .is_some()
809        );
810
811        vault.remove_note("note.md");
812        let changed = handle.reconcile().await.expect("reconcile after removal");
813        assert_eq!(changed, 1);
814        assert!(
815            store
816                .get(VAULT, &note_key("note.md"))
817                .await
818                .unwrap()
819                .is_none()
820        );
821
822        let active = handle.vault.active_vault().await.unwrap();
823        let index_engine = handle.engine_for(&active).await.unwrap();
824        assert_eq!(index_engine.stats().await.indexed_notes, 0);
825    }
826
827    #[tokio::test]
828    async fn hard_reset_forces_a_full_re_embed() {
829        let store = Arc::new(FakeStore::default());
830        let vault = Arc::new(FakeVault::default());
831        vault.put_note("note.md", "Stable content that never changes.");
832        let embedder = Arc::new(CountingEmbedder::new(DIMS));
833        let handle = handle_with(Arc::clone(&store), vault, Arc::clone(&embedder));
834        handle.reconcile().await.expect("initial index");
835        let first_pass = embedder.count();
836        assert!(first_pass > 0);
837
838        // Content did not change, so a plain reconcile would not re-embed;
839        // hard_reset must re-embed anyway.
840        assert_eq!(handle.reconcile().await.unwrap(), 0);
841        let reindexed = handle.hard_reset().await.expect("hard reset");
842        assert_eq!(reindexed, 1);
843        assert_eq!(embedder.count(), first_pass * 2);
844    }
845
846    #[tokio::test]
847    async fn search_tool_rejects_an_empty_query() {
848        let provider = VectorSearchProvider {
849            handle: Arc::new(handle_with(
850                Arc::new(FakeStore::default()),
851                Arc::new(FakeVault::default()),
852                Arc::new(CountingEmbedder::new(DIMS)),
853            )),
854            hooks: HookBus::new(16),
855            shutdown: ShutdownTrigger::new().signal(),
856            worker: Mutex::new(None),
857        };
858        let error = provider
859            .call_tool(
860                "search",
861                json!({"query": "   "}),
862                PluginRequestContext::new("req"),
863            )
864            .await
865            .expect_err("empty query must be rejected");
866        assert_eq!(
867            error.code,
868            turbovault_plugin_api::PluginErrorCode::InvalidInput
869        );
870    }
871
872    #[tokio::test]
873    async fn status_reports_before_and_after_the_engine_is_built() {
874        let vault = Arc::new(FakeVault::default());
875        vault.put_note("note.md", "Searchable content for status reporting.");
876        let provider = VectorSearchProvider {
877            handle: Arc::new(handle_with(
878                Arc::new(FakeStore::default()),
879                vault,
880                Arc::new(CountingEmbedder::new(DIMS)),
881            )),
882            hooks: HookBus::new(16),
883            shutdown: ShutdownTrigger::new().signal(),
884            worker: Mutex::new(None),
885        };
886
887        let before = provider
888            .call_tool("status", json!({}), PluginRequestContext::new("req"))
889            .await
890            .expect("status before build");
891        let before = before.structured_content.expect("structured content");
892        assert_eq!(before["indexed_notes"], 0);
893
894        provider
895            .call_tool(
896                "search",
897                json!({"query": "content"}),
898                PluginRequestContext::new("req"),
899            )
900            .await
901            .expect("search builds the engine");
902
903        let after = provider
904            .call_tool("status", json!({}), PluginRequestContext::new("req"))
905            .await
906            .expect("status after build");
907        let after = after.structured_content.expect("structured content");
908        assert_eq!(after["indexed_notes"], 1);
909        assert_eq!(after["dims"], DIMS);
910    }
911
912    #[tokio::test]
913    async fn worker_reconciles_on_a_hook_event_and_stops_on_shutdown() {
914        let vault = Arc::new(FakeVault::default());
915        vault.put_note("note.md", "Indexed before the worker starts.");
916        let hooks = HookBus::new(16);
917        let trigger = ShutdownTrigger::new();
918        let provider = VectorSearchProvider {
919            handle: Arc::new(handle_with(
920                Arc::new(FakeStore::default()),
921                Arc::clone(&vault),
922                Arc::new(CountingEmbedder::new(DIMS)),
923            )),
924            hooks: hooks.clone(),
925            shutdown: trigger.signal(),
926            worker: Mutex::new(None),
927        };
928
929        provider.start().await.expect("start");
930        vault.put_note("second.md", "Indexed after a hook event.");
931        hooks
932            .publish(
933                VAULT,
934                turbovault_plugin_api::HookEvent::FileCreated {
935                    path: "second.md".to_string(),
936                },
937                None,
938                None,
939                turbovault_plugin_api::EventAttribution::ExternalOrUnknown,
940            )
941            .expect("publish");
942
943        // The worker's reconcile is async and not itself observable from
944        // here, so poll for it instead of asserting instantly. Bounded by
945        // yield count rather than a wall-clock deadline, and checking engine
946        // state directly rather than through `call_tool`: nothing here waits
947        // on real I/O or a timer (the fakes resolve immediately), and a cheap
948        // check keeps the bound meaningful even when the host is under heavy
949        // unrelated load, instead of the iteration budget being consumed by
950        // JSON serialization work on every poll.
951        let mut indexed = 0;
952        for _ in 0..1_000 {
953            if let Some((_, index_engine)) = provider.handle.engine.get() {
954                indexed = index_engine.stats().await.indexed_notes;
955                if indexed == 2 {
956                    break;
957                }
958            }
959            tokio::task::yield_now().await;
960        }
961        assert_eq!(indexed, 2, "worker did not reconcile the hook event");
962
963        // The real host signals the trigger, then awaits each plugin's
964        // shutdown (see `ObsidianMcpServer::shutdown`); `PluginProvider::shutdown`
965        // only awaits cleanup; it does not itself fire the signal the worker
966        // loop is blocked on.
967        trigger.shutdown();
968        provider.shutdown().await;
969        assert!(provider.worker.lock().await.is_none());
970    }
971
972    #[tokio::test]
973    async fn plugin_descriptor_is_a_stable_namespace() {
974        let plugin = VectorSearchPlugin::with_embedder_factory(|_| {
975            Ok(Arc::new(DeterministicEmbedder::new(DIMS)) as Arc<dyn EmbeddingEngine>)
976        });
977        assert_eq!(plugin.descriptor().id, "vector_search");
978    }
979}