sift_core/index/
store.rs

1use std::path::{Path, PathBuf};
2
3use super::snapshot::{
4    DiskSnapshotStore, SnapshotId, SnapshotLease, SnapshotManifest, SnapshotRead, SnapshotStore,
5    SnapshotWrite, SnapshotWriterSession,
6};
7use super::{CorpusKind, IndexError};
8
9/// Index lifecycle orchestrator backed by a [`SnapshotStore`] for atomic
10/// persistence and coordination.
11pub struct IndexStore<S: SnapshotStore = DiskSnapshotStore> {
12    snapshots: S,
13    sift_dir: PathBuf,
14    meta: Option<super::meta::StoreMeta>,
15}
16
17impl IndexStore<DiskSnapshotStore> {
18    /// Open an existing store at `sift_dir`.
19    ///
20    /// # Errors
21    ///
22    /// Returns an error if `CURRENT` exists but cannot be read.
23    pub fn open(sift_dir: &Path) -> crate::Result<Self> {
24        let snapshots = DiskSnapshotStore::open(sift_dir)?;
25        let meta = super::meta::StoreMeta::read(sift_dir).ok();
26        Ok(Self {
27            snapshots,
28            sift_dir: sift_dir.to_path_buf(),
29            meta,
30        })
31    }
32
33    /// Open an existing store or create a new one at `sift_dir`.
34    ///
35    /// # Errors
36    ///
37    /// Returns an error if the store directory cannot be created or metadata
38    /// cannot be written.
39    pub fn open_or_create(
40        sift_dir: &Path,
41        root: &Path,
42        corpus_kind: CorpusKind,
43        follow_links: bool,
44        indexes: &[super::IndexKind],
45    ) -> crate::Result<Self> {
46        std::fs::create_dir_all(sift_dir)?;
47
48        if !super::meta::StoreMeta::path(sift_dir).exists() {
49            let _guard = acquire_write_lock(sift_dir)?;
50            if !super::meta::StoreMeta::path(sift_dir).exists() {
51                let canonical_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
52                let meta = super::meta::StoreMeta::new(
53                    canonical_root,
54                    corpus_kind,
55                    follow_links,
56                    indexes.to_vec(),
57                );
58                meta.write(sift_dir)?;
59            }
60        }
61
62        Self::open(sift_dir)
63    }
64
65    #[must_use]
66    pub fn snapshot_dir(&self, id: &str) -> PathBuf {
67        self.sift_dir.join("snapshots").join(id)
68    }
69
70    // ------------------------------------------------------------------
71    // Read path
72    // ------------------------------------------------------------------
73
74    /// Open the current snapshot, returning an immutable [`Snapshot`] with
75    /// its indexes opened and a reader lease held.
76    ///
77    /// Re-reads `CURRENT` from disk to ensure freshness. Retries once if
78    /// the snapshot disappears during opening.
79    ///
80    /// # Errors
81    ///
82    /// Returns an error if the manifest is malformed, an index kind is
83    /// unknown, or the snapshot could not be opened after retry.
84    pub(crate) fn open_current(&self) -> crate::Result<super::snapshot::Snapshot> {
85        for attempt in 0..2 {
86            let Some(current_id) = DiskSnapshotStore::read_current_id(&self.sift_dir)? else {
87                return Ok(super::snapshot::Snapshot::empty(PathBuf::new()));
88            };
89
90            let Some(ref meta) = self.meta else {
91                return Err(crate::Error::Index(IndexError::Io {
92                    path: self.sift_dir.join("sift.meta"),
93                    source: std::io::Error::new(
94                        std::io::ErrorKind::NotFound,
95                        "store metadata not found",
96                    ),
97                }));
98            };
99            let root = meta.root.clone();
100            let corpus_kind = meta.corpus_kind;
101
102            let snap_dir = self.sift_dir.join("snapshots").join(&current_id);
103
104            let lease = SnapshotLease::create_file(&self.sift_dir, &current_id)?;
105
106            if !snap_dir.exists() {
107                drop(lease);
108                continue;
109            }
110
111            let manifest_path = snap_dir.join("manifest.json");
112            let manifest_raw = match std::fs::read_to_string(&manifest_path) {
113                Ok(raw) => raw,
114                Err(e) if e.kind() == std::io::ErrorKind::NotFound && attempt == 0 => {
115                    drop(lease);
116                    continue;
117                }
118                Err(e) => return Err(crate::Error::Io(e)),
119            };
120
121            let manifest: SnapshotManifest = serde_json::from_str(&manifest_raw).map_err(|e| {
122                crate::Error::Index(IndexError::InvalidManifest {
123                    path: manifest_path.clone(),
124                    source: e,
125                })
126            })?;
127
128            let mut indexes = Vec::new();
129            for name in &manifest.indexes {
130                let kind: super::IndexKind = name
131                    .parse()
132                    .map_err(|_| crate::Error::Index(IndexError::UnknownIndexKind(name.clone())))?;
133                let index_dir = snap_dir.join(name);
134                indexes.push(kind.open(
135                    super::IndexSource::Directory(&index_dir),
136                    &root,
137                    corpus_kind,
138                )?);
139            }
140
141            return Ok(super::snapshot::Snapshot::current(root, indexes, lease));
142        }
143
144        Err(crate::Error::Io(std::io::Error::new(
145            std::io::ErrorKind::NotFound,
146            "snapshot disappeared during open",
147        )))
148    }
149}
150
151// ------------------------------------------------------------------
152// Generic impl (works for any SnapshotStore)
153// ------------------------------------------------------------------
154
155impl<S: SnapshotStore> IndexStore<S> {
156    #[must_use]
157    pub fn current_id(&self) -> Option<&str> {
158        self.snapshots.current_id().map(SnapshotId::as_str)
159    }
160
161    // ------------------------------------------------------------------
162    // Write path
163    // ------------------------------------------------------------------
164
165    /// Build a new snapshot using the given index kinds.
166    ///
167    /// Acquires the writer session, builds each index kind as artifacts, and
168    /// publishes the snapshot.
169    ///
170    /// Returns the snapshot id.
171    ///
172    /// # Errors
173    ///
174    /// Returns an error if the writer session cannot be acquired, the index
175    /// build fails, or publishing fails.
176    pub fn build(
177        &mut self,
178        kinds: &[super::IndexKind],
179        config: &super::IndexConfig<'_>,
180    ) -> crate::Result<String> {
181        let mut writer = self.snapshots.writer()?;
182        let mut txn = writer.begin()?;
183
184        for kind in kinds {
185            kind.build(
186                config,
187                super::IndexDestination::Snapshot {
188                    writer: &mut txn,
189                    namespace: kind.as_str(),
190                },
191            )?;
192        }
193
194        let manifest = SnapshotManifest {
195            id: txn.id().clone(),
196            indexes: kinds.iter().map(|k| k.as_str().to_string()).collect(),
197        };
198        let id = writer.publish(txn, manifest)?;
199        Ok(id.to_string())
200    }
201
202    /// Update the current snapshot, rebuilding only indexes whose corpus
203    /// changed.
204    ///
205    /// Acquires the writer session, checks for changes, copies unchanged
206    /// artifacts from the current snapshot, and publishes a new one.
207    ///
208    /// Returns the snapshot id if a new snapshot was published, or `None` if
209    /// no index changed.
210    ///
211    /// # Errors
212    ///
213    /// Returns an error if the writer session cannot be acquired, the current
214    /// snapshot cannot be opened, the update check fails, or publishing fails.
215    pub fn update(
216        &mut self,
217        kinds: &[super::IndexKind],
218        config: &super::IndexConfig<'_>,
219    ) -> crate::Result<Option<String>> {
220        let mut writer = self.snapshots.writer()?;
221
222        let Some(current) = writer.current()? else {
223            // No current snapshot — build from scratch.
224            let mut txn = writer.begin()?;
225            for kind in kinds {
226                kind.build(
227                    config,
228                    super::IndexDestination::Snapshot {
229                        writer: &mut txn,
230                        namespace: kind.as_str(),
231                    },
232                )?;
233            }
234            let manifest = SnapshotManifest {
235                id: txn.id().clone(),
236                indexes: kinds.iter().map(|k| k.as_str().to_string()).collect(),
237            };
238            let id = writer.publish(txn, manifest)?;
239            return Ok(Some(id.to_string()));
240        };
241
242        let mut txn = writer.begin()?;
243
244        let changed: Vec<bool> = kinds
245            .iter()
246            .map(|kind| {
247                kind.update(
248                    super::IndexSource::Snapshot {
249                        reader: &current as &dyn super::snapshot::SnapshotRead,
250                        namespace: kind.as_str(),
251                    },
252                    config,
253                    super::IndexDestination::Snapshot {
254                        writer: &mut txn,
255                        namespace: kind.as_str(),
256                    },
257                )
258            })
259            .collect::<crate::Result<_>>()?;
260
261        if !changed.iter().any(|&c| c) {
262            return Ok(None);
263        }
264
265        for (kind, did_change) in kinds.iter().zip(&changed) {
266            if !did_change {
267                for artifact_name in kind.artifact_names() {
268                    let data = current.artifact(kind.as_str(), artifact_name)?;
269                    let bytes = data.as_ref().to_vec();
270                    txn.put_artifact(kind.as_str(), artifact_name, bytes)?;
271                }
272            }
273        }
274
275        let manifest = SnapshotManifest {
276            id: txn.id().clone(),
277            indexes: kinds.iter().map(|k| k.as_str().to_string()).collect(),
278        };
279        let id = writer.publish(txn, manifest)?;
280        Ok(Some(id.to_string()))
281    }
282}
283fn acquire_write_lock(sift_dir: &Path) -> crate::Result<WriteLockGuard> {
284    let lock_path = sift_dir.join("write.lock");
285    let mut lock_file = fslock::LockFile::open(&lock_path)?;
286    lock_file.lock()?;
287    Ok(WriteLockGuard { _file: lock_file })
288}
289
290/// Guard that releases the write lock when dropped.
291struct WriteLockGuard {
292    _file: fslock::LockFile,
293}
294
295#[cfg(test)]
296mod tests {
297    use super::*;
298    use crate::index::meta::StoreMeta;
299    use crate::index::{CorpusKind, CorpusSpec, IndexConfig, IndexKind};
300    use crate::search::filter::VisibilityConfig;
301    use std::fs;
302    use tempfile::TempDir;
303
304    const MANIFEST_FILE: &str = "manifest.json";
305
306    #[test]
307    fn build_creates_store_layout() {
308        let tmp = TempDir::new().expect("create temp dir");
309        let corpus = tmp.path().join("corpus");
310        fs::create_dir_all(&corpus).expect("create corpus");
311        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
312
313        let sift_dir = tmp.path().join(".sift");
314        let mut store = IndexStore::open_or_create(
315            &sift_dir,
316            &corpus,
317            CorpusKind::Directory,
318            false,
319            &[IndexKind::Trigram],
320        )
321        .expect("open store");
322
323        store
324            .build(
325                &[IndexKind::Trigram],
326                &IndexConfig {
327                    corpus: CorpusSpec {
328                        root: &corpus,
329                        kind: CorpusKind::Directory,
330                        follow_links: false,
331                        include_paths: &[],
332                        exclude_paths: &[],
333                    },
334                    visibility: VisibilityConfig::default(),
335                },
336            )
337            .expect("build");
338
339        assert!(StoreMeta::path(&sift_dir).exists());
340
341        let id = store.current_id().expect("has current id");
342        let snapshot_dir = store.snapshot_dir(id);
343        assert!(snapshot_dir.exists());
344        assert!(snapshot_dir.join(MANIFEST_FILE).exists());
345        assert!(snapshot_dir.join("trigram").exists());
346        assert!(snapshot_dir.join("trigram").join("files.bin").exists());
347        assert!(snapshot_dir.join("trigram").join("lexicon.bin").exists());
348        assert!(snapshot_dir.join("trigram").join("postings.bin").exists());
349        assert!(snapshot_dir.join("trigram").join("trigrams.bin").exists());
350
351        assert!(!id.is_empty());
352    }
353
354    #[test]
355    fn open_current_returns_indexes() {
356        let tmp = TempDir::new().expect("create temp dir");
357        let corpus = tmp.path().join("corpus");
358        fs::create_dir_all(&corpus).expect("create corpus");
359        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
360
361        let sift_dir = tmp.path().join(".sift");
362        let mut store = IndexStore::open_or_create(
363            &sift_dir,
364            &corpus,
365            CorpusKind::Directory,
366            false,
367            &[IndexKind::Trigram],
368        )
369        .expect("open store");
370
371        store
372            .build(
373                &[IndexKind::Trigram],
374                &IndexConfig {
375                    corpus: CorpusSpec {
376                        root: &corpus,
377                        kind: CorpusKind::Directory,
378                        follow_links: false,
379                        include_paths: &[],
380                        exclude_paths: &[],
381                    },
382                    visibility: VisibilityConfig::default(),
383                },
384            )
385            .expect("build");
386
387        drop(store);
388        let store = IndexStore::open(&sift_dir).expect("reopen store");
389        let snapshot = store.open_current().expect("open current");
390        assert_eq!(snapshot.indexes().len(), 1);
391        let canon_corpus = corpus.canonicalize().unwrap();
392        assert_eq!(snapshot.indexes()[0].root(), &canon_corpus);
393        assert!(store.current_id().is_some(), "store should have current id");
394    }
395
396    #[test]
397    fn open_returns_empty_when_no_current() {
398        let tmp = TempDir::new().expect("create temp dir");
399        let sift_dir = tmp.path().join(".sift");
400        std::fs::create_dir_all(&sift_dir).expect("create sift dir");
401
402        let store = IndexStore::open(&sift_dir).expect("open store");
403        assert!(store.current_id().is_none());
404        let snapshot = store.open_current().expect("open current");
405        assert!(snapshot.is_empty());
406    }
407
408    #[test]
409    fn open_without_sift_dir_returns_empty() {
410        let tmp = TempDir::new().expect("create temp dir");
411        let store = IndexStore::open(&tmp.path().join(".sift")).expect("open store");
412        assert!(store.current_id().is_none());
413    }
414
415    #[test]
416    fn update_skips_rebuild_when_unchanged() {
417        let tmp = TempDir::new().expect("create temp dir");
418        let corpus = tmp.path().join("corpus");
419        fs::create_dir_all(&corpus).expect("create corpus");
420        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
421
422        let sift_dir = tmp.path().join(".sift");
423        let config = IndexConfig {
424            corpus: CorpusSpec {
425                root: &corpus,
426                kind: CorpusKind::Directory,
427                follow_links: false,
428                include_paths: &[],
429                exclude_paths: &[],
430            },
431            visibility: VisibilityConfig::default(),
432        };
433
434        let mut store = IndexStore::open_or_create(
435            &sift_dir,
436            &corpus,
437            CorpusKind::Directory,
438            false,
439            &[IndexKind::Trigram],
440        )
441        .expect("open store");
442        store.build(&[IndexKind::Trigram], &config).expect("build");
443
444        let id_after_build = store.current_id().expect("has id").to_string();
445
446        let changed = store
447            .update(&[IndexKind::Trigram], &config)
448            .expect("update");
449        assert_eq!(changed, None, "expected no rebuild when corpus unchanged");
450        assert_eq!(store.current_id().unwrap(), id_after_build);
451    }
452
453    #[test]
454    fn update_rebuilds_when_file_added() {
455        let tmp = TempDir::new().expect("create temp dir");
456        let corpus = tmp.path().join("corpus");
457        fs::create_dir_all(&corpus).expect("create corpus");
458        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
459
460        let sift_dir = tmp.path().join(".sift");
461        let config = IndexConfig {
462            corpus: CorpusSpec {
463                root: &corpus,
464                kind: CorpusKind::Directory,
465                follow_links: false,
466                include_paths: &[],
467                exclude_paths: &[],
468            },
469            visibility: VisibilityConfig::default(),
470        };
471
472        let mut store = IndexStore::open_or_create(
473            &sift_dir,
474            &corpus,
475            CorpusKind::Directory,
476            false,
477            &[IndexKind::Trigram],
478        )
479        .expect("open store");
480        store.build(&[IndexKind::Trigram], &config).expect("build");
481
482        let id_after_build = store.current_id().expect("has id").to_string();
483
484        fs::write(corpus.join("g.txt"), "new file\n").expect("write new file");
485
486        let changed = store
487            .update(&[IndexKind::Trigram], &config)
488            .expect("update");
489        assert!(changed.is_some(), "expected rebuild when file added");
490        assert_ne!(store.current_id().unwrap(), id_after_build);
491    }
492
493    #[test]
494    fn update_builds_when_no_current_snapshot() {
495        let tmp = TempDir::new().expect("create temp dir");
496        let corpus = tmp.path().join("corpus");
497        fs::create_dir_all(&corpus).expect("create corpus");
498        fs::write(corpus.join("f.txt"), "hello\n").expect("write file");
499
500        let sift_dir = tmp.path().join(".sift");
501        let config = IndexConfig {
502            corpus: CorpusSpec {
503                root: &corpus,
504                kind: CorpusKind::Directory,
505                follow_links: false,
506                include_paths: &[],
507                exclude_paths: &[],
508            },
509            visibility: VisibilityConfig::default(),
510        };
511
512        let mut store = IndexStore::open_or_create(
513            &sift_dir,
514            &corpus,
515            CorpusKind::Directory,
516            false,
517            &[IndexKind::Trigram],
518        )
519        .expect("open store");
520
521        let changed = store
522            .update(&[IndexKind::Trigram], &config)
523            .expect("update");
524        assert!(changed.is_some(), "expected build when no snapshot exists");
525        assert!(store.current_id().is_some());
526    }
527
528    #[test]
529    fn build_acquires_write_lock() {
530        let tmp = TempDir::new().expect("create temp dir");
531        let corpus = tmp.path().join("corpus");
532        fs::create_dir_all(&corpus).expect("create corpus");
533        fs::write(corpus.join("f.txt"), "content\n").expect("write file");
534
535        let sift_dir = tmp.path().join(".sift");
536        let config = IndexConfig {
537            corpus: CorpusSpec {
538                root: &corpus,
539                kind: CorpusKind::Directory,
540                follow_links: false,
541                include_paths: &[],
542                exclude_paths: &[],
543            },
544            visibility: VisibilityConfig::default(),
545        };
546
547        let mut store = IndexStore::open_or_create(
548            &sift_dir,
549            &corpus,
550            CorpusKind::Directory,
551            false,
552            &[IndexKind::Trigram],
553        )
554        .expect("open store");
555
556        store.build(&[IndexKind::Trigram], &config).expect("build");
557
558        // Verify lock was released after build by acquiring it externally.
559        let lock_path = sift_dir.join("write.lock");
560        let mut lock = fslock::LockFile::open(&lock_path).expect("open lock");
561        assert!(
562            lock.try_lock().expect("try lock"),
563            "write lock should be released after build"
564        );
565        drop(lock);
566    }
567
568    #[test]
569    fn writer_refreshes_snapshot_state_on_acquire() {
570        let tmp = TempDir::new().expect("create temp dir");
571        let corpus = tmp.path().join("corpus");
572        fs::create_dir_all(&corpus).expect("create corpus");
573        fs::write(corpus.join("f.txt"), "hello\n").expect("write file");
574
575        let sift_dir = tmp.path().join(".sift");
576        let config = IndexConfig {
577            corpus: CorpusSpec {
578                root: &corpus,
579                kind: CorpusKind::Directory,
580                follow_links: false,
581                include_paths: &[],
582                exclude_paths: &[],
583            },
584            visibility: VisibilityConfig::default(),
585        };
586
587        // Store A builds.
588        let mut store_a = IndexStore::open_or_create(
589            &sift_dir,
590            &corpus,
591            CorpusKind::Directory,
592            false,
593            &[IndexKind::Trigram],
594        )
595        .expect("open store");
596        store_a
597            .build(&[IndexKind::Trigram], &config)
598            .expect("build");
599
600        // Store A publishes a new snapshot.
601        fs::write(corpus.join("g.txt"), "new\n").expect("write");
602        store_a
603            .update(&[IndexKind::Trigram], &config)
604            .expect("update");
605
606        // Store B opens — it should see the same current (reads fresh from disk
607        // at open time, then caches).
608        let mut store_b = IndexStore::open(&sift_dir).expect("open store");
609        assert_eq!(
610            store_b.current_id(),
611            store_a.current_id(),
612            "freshly-opened store_b should see store_a's latest snapshot"
613        );
614
615        // Store B acquires the write lock for its own update, which refreshes
616        // its snapshot state.  Since the corpus is unchanged, no new snapshot
617        // is published and current_id stays the same.
618        let changed = store_b
619            .update(&[IndexKind::Trigram], &config)
620            .expect("update");
621        assert_eq!(changed, None, "corpus unchanged after store_a update");
622        assert_eq!(
623            store_b.current_id(),
624            store_a.current_id(),
625            "store_b should still point to the same snapshot"
626        );
627    }
628}
sift_core/index/store.rs

sift_core/index/
store.rs
