sift_core/index/
store.rs

1use std::path::{Path, PathBuf};
2
3use serde::{Deserialize, Serialize};
4
5use super::meta::StoreMeta;
6use super::snapshot::{self, SnapshotStore};
7use super::{CorpusKind, IndexError};
8
9const MANIFEST_FILE: &str = "manifest.json";
10
11/// Manifest written into each snapshot directory listing the indexes it
12/// contains.
13#[derive(Debug, Serialize, Deserialize)]
14struct SnapshotManifest {
15    id: String,
16    indexes: Vec<String>,
17}
18
19/// Index lifecycle orchestrator backed by a [`SnapshotStore`] for atomic
20/// persistence and [`StoreMeta`] for corpus configuration.
21pub struct IndexStore {
22    snapshots: SnapshotStore,
23    sift_dir: PathBuf,
24}
25
26impl IndexStore {
27    /// Open an existing store at `sift_dir`.
28    ///
29    /// # Errors
30    ///
31    /// Returns an error if `CURRENT` exists but cannot be read.
32    pub fn open(sift_dir: &Path) -> crate::Result<Self> {
33        let snapshots = SnapshotStore::open(sift_dir)?;
34        Ok(Self {
35            snapshots,
36            sift_dir: sift_dir.to_path_buf(),
37        })
38    }
39
40    /// Open an existing store or create a new one at `sift_dir`.
41    ///
42    /// # Errors
43    ///
44    /// Returns an error if the store directory cannot be created or metadata
45    /// cannot be written.
46    pub fn open_or_create(
47        sift_dir: &Path,
48        root: &Path,
49        corpus_kind: CorpusKind,
50        follow_links: bool,
51        indexes: &[super::IndexKind],
52    ) -> crate::Result<Self> {
53        std::fs::create_dir_all(sift_dir)?;
54
55        if !StoreMeta::path(sift_dir).exists() {
56            let canonical_root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
57            let meta = StoreMeta::new(canonical_root, corpus_kind, follow_links, indexes.to_vec());
58            meta.write(sift_dir)?;
59        }
60
61        Self::open(sift_dir)
62    }
63
64    #[must_use]
65    pub fn current_id(&self) -> Option<&str> {
66        self.snapshots.current_id()
67    }
68
69    #[must_use]
70    pub fn snapshot_dir(&self, id: &str) -> PathBuf {
71        self.snapshots.current_dir().map_or_else(
72            || self.sift_dir.join("snapshots").join(id),
73            |d| {
74                let parent = d.parent().unwrap_or(&self.sift_dir);
75                parent.join(id)
76            },
77        )
78    }
79
80    /// Build a new snapshot using the given index kinds.
81    ///
82    /// Returns the snapshot id.
83    ///
84    /// # Errors
85    ///
86    /// Returns an error if the index build fails, the manifest cannot be
87    /// written, or snapshot commit fails.
88    pub fn build(
89        &mut self,
90        kinds: &[super::IndexKind],
91        config: &super::IndexConfig<'_>,
92    ) -> crate::Result<String> {
93        let snapshot = self.snapshots.begin()?;
94
95        for kind in kinds {
96            let index_dir = snapshot.dir().join(kind.as_str());
97            std::fs::create_dir_all(&index_dir)?;
98            kind.build(config, &index_dir)?;
99        }
100
101        Self::write_manifest(&snapshot, kinds)?;
102        let id = snapshot.id().to_string();
103        self.snapshots.commit(snapshot)?;
104        Ok(id)
105    }
106
107    /// Update the current snapshot, rebuilding only indexes whose corpus
108    /// changed.
109    ///
110    /// Returns the snapshot id if a new snapshot was published, or `None` if
111    /// no index changed.
112    ///
113    /// # Errors
114    ///
115    /// Returns an error if the current snapshot cannot be opened, the update
116    /// check fails, or publishing the new snapshot fails.
117    pub fn update(
118        &mut self,
119        kinds: &[super::IndexKind],
120        config: &super::IndexConfig<'_>,
121    ) -> crate::Result<Option<String>> {
122        let Some(current_dir) = self.snapshots.current_dir() else {
123            let id = self.build(kinds, config)?;
124            return Ok(Some(id));
125        };
126
127        let snapshot = self.snapshots.begin()?;
128
129        let changed: Vec<bool> = kinds
130            .iter()
131            .map(|kind| kind.update(&current_dir, config, &snapshot.dir().join(kind.as_str())))
132            .collect::<crate::Result<_>>()?;
133
134        if !changed.iter().any(|&c| c) {
135            return Ok(None);
136        }
137
138        for (kind, did_change) in kinds.iter().zip(&changed) {
139            if !did_change {
140                let src = current_dir.join(kind.as_str());
141                if src.exists() {
142                    snapshot::copy_dir_contents(&src, &snapshot.dir().join(kind.as_str()))?;
143                }
144            }
145        }
146
147        Self::write_manifest(&snapshot, kinds)?;
148        let id = snapshot.id().to_string();
149        self.snapshots.commit(snapshot)?;
150        Ok(Some(id))
151    }
152
153    /// Open all indexes in the current snapshot.
154    ///
155    /// Returns an empty vector if no snapshot exists.
156    ///
157    /// # Errors
158    ///
159    /// Returns an error if the manifest is malformed or an index kind is
160    /// unknown.
161    pub fn open_current(&self) -> crate::Result<Vec<super::Index>> {
162        let Some(snapshot_dir) = self.snapshots.current_dir() else {
163            return Ok(Vec::new());
164        };
165
166        let manifest_path = snapshot_dir.join(MANIFEST_FILE);
167        let manifest_raw = std::fs::read_to_string(&manifest_path)?;
168        let manifest: SnapshotManifest = serde_json::from_str(&manifest_raw).map_err(|e| {
169            crate::Error::Index(IndexError::InvalidManifest {
170                path: manifest_path.clone(),
171                source: e,
172            })
173        })?;
174
175        let meta = StoreMeta::read(&self.sift_dir)?;
176
177        let mut indexes = Vec::new();
178        for name in &manifest.indexes {
179            let kind: super::IndexKind = name
180                .parse()
181                .map_err(|_| crate::Error::Index(IndexError::UnknownIndexKind(name.clone())))?;
182            let index_dir = snapshot_dir.join(name);
183            indexes.push(kind.open_from_dir(&index_dir, &meta.root, meta.corpus_kind)?);
184        }
185
186        Ok(indexes)
187    }
188
189    fn write_manifest(
190        snapshot: &super::snapshot::Snapshot,
191        kinds: &[super::IndexKind],
192    ) -> crate::Result<()> {
193        let manifest = SnapshotManifest {
194            id: snapshot.id().to_string(),
195            indexes: kinds.iter().map(|k| k.as_str().to_string()).collect(),
196        };
197        let json = serde_json::to_vec_pretty(&manifest).map_err(|e| {
198            crate::Error::Index(IndexError::InvalidManifest {
199                path: snapshot.dir().join(MANIFEST_FILE),
200                source: e,
201            })
202        })?;
203        std::fs::write(snapshot.dir().join(MANIFEST_FILE), json)?;
204        Ok(())
205    }
206}
207
208#[cfg(test)]
209mod tests {
210    use super::*;
211    use crate::index::{CorpusKind, CorpusSpec, IndexConfig, IndexKind};
212    use crate::search::filter::VisibilityConfig;
213    use std::fs;
214    use tempfile::TempDir;
215
216    #[test]
217    fn build_creates_store_layout() {
218        let tmp = TempDir::new().expect("create temp dir");
219        let corpus = tmp.path().join("corpus");
220        fs::create_dir_all(&corpus).expect("create corpus");
221        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
222
223        let sift_dir = tmp.path().join(".sift");
224        let mut store = IndexStore::open_or_create(
225            &sift_dir,
226            &corpus,
227            CorpusKind::Directory,
228            false,
229            &[IndexKind::Trigram],
230        )
231        .expect("open store");
232
233        store
234            .build(
235                &[IndexKind::Trigram],
236                &IndexConfig {
237                    corpus: CorpusSpec {
238                        root: &corpus,
239                        kind: CorpusKind::Directory,
240                        follow_links: false,
241                        include_paths: &[],
242                        exclude_paths: &[],
243                    },
244                    visibility: VisibilityConfig::default(),
245                },
246            )
247            .expect("build");
248
249        assert!(StoreMeta::path(&sift_dir).exists());
250
251        let id = store.current_id().expect("has current id");
252        let snapshot_dir = store.snapshots.current_dir().expect("has snapshot dir");
253        assert!(snapshot_dir.exists());
254        assert!(snapshot_dir.join(MANIFEST_FILE).exists());
255        assert!(snapshot_dir.join("trigram").exists());
256        assert!(snapshot_dir.join("trigram").join("files.bin").exists());
257        assert!(snapshot_dir.join("trigram").join("lexicon.bin").exists());
258        assert!(snapshot_dir.join("trigram").join("postings.bin").exists());
259        assert!(snapshot_dir.join("trigram").join("trigrams.bin").exists());
260
261        assert!(!id.is_empty());
262    }
263
264    #[test]
265    fn open_current_returns_indexes() {
266        let tmp = TempDir::new().expect("create temp dir");
267        let corpus = tmp.path().join("corpus");
268        fs::create_dir_all(&corpus).expect("create corpus");
269        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
270
271        let sift_dir = tmp.path().join(".sift");
272        let mut store = IndexStore::open_or_create(
273            &sift_dir,
274            &corpus,
275            CorpusKind::Directory,
276            false,
277            &[IndexKind::Trigram],
278        )
279        .expect("open store");
280
281        store
282            .build(
283                &[IndexKind::Trigram],
284                &IndexConfig {
285                    corpus: CorpusSpec {
286                        root: &corpus,
287                        kind: CorpusKind::Directory,
288                        follow_links: false,
289                        include_paths: &[],
290                        exclude_paths: &[],
291                    },
292                    visibility: VisibilityConfig::default(),
293                },
294            )
295            .expect("build");
296
297        drop(store);
298        let store = IndexStore::open(&sift_dir).expect("reopen store");
299        let indexes = store.open_current().expect("open current");
300        assert_eq!(indexes.len(), 1);
301        let canon_corpus = corpus.canonicalize().unwrap();
302        assert_eq!(indexes[0].root(), &canon_corpus);
303    }
304
305    #[test]
306    fn open_returns_empty_when_no_current() {
307        let tmp = TempDir::new().expect("create temp dir");
308        let sift_dir = tmp.path().join(".sift");
309        std::fs::create_dir_all(&sift_dir).expect("create sift dir");
310
311        let store = IndexStore::open(&sift_dir).expect("open store");
312        assert!(store.current_id().is_none());
313        let indexes = store.open_current().expect("open current");
314        assert!(indexes.is_empty());
315    }
316
317    #[test]
318    fn open_without_sift_dir_returns_empty() {
319        let tmp = TempDir::new().expect("create temp dir");
320        let store = IndexStore::open(&tmp.path().join(".sift")).expect("open store");
321        assert!(store.current_id().is_none());
322    }
323
324    #[test]
325    fn update_skips_rebuild_when_unchanged() {
326        let tmp = TempDir::new().expect("create temp dir");
327        let corpus = tmp.path().join("corpus");
328        fs::create_dir_all(&corpus).expect("create corpus");
329        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
330
331        let sift_dir = tmp.path().join(".sift");
332        let config = IndexConfig {
333            corpus: CorpusSpec {
334                root: &corpus,
335                kind: CorpusKind::Directory,
336                follow_links: false,
337                include_paths: &[],
338                exclude_paths: &[],
339            },
340            visibility: VisibilityConfig::default(),
341        };
342
343        let mut store = IndexStore::open_or_create(
344            &sift_dir,
345            &corpus,
346            CorpusKind::Directory,
347            false,
348            &[IndexKind::Trigram],
349        )
350        .expect("open store");
351        store.build(&[IndexKind::Trigram], &config).expect("build");
352
353        let id_after_build = store.current_id().expect("has id").to_string();
354
355        let changed = store
356            .update(&[IndexKind::Trigram], &config)
357            .expect("update");
358        assert_eq!(changed, None, "expected no rebuild when corpus unchanged");
359        assert_eq!(store.current_id().unwrap(), id_after_build);
360    }
361
362    #[test]
363    fn update_rebuilds_when_file_added() {
364        let tmp = TempDir::new().expect("create temp dir");
365        let corpus = tmp.path().join("corpus");
366        fs::create_dir_all(&corpus).expect("create corpus");
367        fs::write(corpus.join("f.txt"), "hello world\n").expect("write file");
368
369        let sift_dir = tmp.path().join(".sift");
370        let config = IndexConfig {
371            corpus: CorpusSpec {
372                root: &corpus,
373                kind: CorpusKind::Directory,
374                follow_links: false,
375                include_paths: &[],
376                exclude_paths: &[],
377            },
378            visibility: VisibilityConfig::default(),
379        };
380
381        let mut store = IndexStore::open_or_create(
382            &sift_dir,
383            &corpus,
384            CorpusKind::Directory,
385            false,
386            &[IndexKind::Trigram],
387        )
388        .expect("open store");
389        store.build(&[IndexKind::Trigram], &config).expect("build");
390
391        let id_after_build = store.current_id().expect("has id").to_string();
392
393        fs::write(corpus.join("g.txt"), "new file\n").expect("write new file");
394
395        let changed = store
396            .update(&[IndexKind::Trigram], &config)
397            .expect("update");
398        assert!(changed.is_some(), "expected rebuild when file added");
399        assert_ne!(store.current_id().unwrap(), id_after_build);
400    }
401
402    #[test]
403    fn update_builds_when_no_current_snapshot() {
404        let tmp = TempDir::new().expect("create temp dir");
405        let corpus = tmp.path().join("corpus");
406        fs::create_dir_all(&corpus).expect("create corpus");
407        fs::write(corpus.join("f.txt"), "hello\n").expect("write file");
408
409        let sift_dir = tmp.path().join(".sift");
410        let config = IndexConfig {
411            corpus: CorpusSpec {
412                root: &corpus,
413                kind: CorpusKind::Directory,
414                follow_links: false,
415                include_paths: &[],
416                exclude_paths: &[],
417            },
418            visibility: VisibilityConfig::default(),
419        };
420
421        let mut store = IndexStore::open_or_create(
422            &sift_dir,
423            &corpus,
424            CorpusKind::Directory,
425            false,
426            &[IndexKind::Trigram],
427        )
428        .expect("open store");
429
430        let changed = store
431            .update(&[IndexKind::Trigram], &config)
432            .expect("update");
433        assert!(changed.is_some(), "expected build when no snapshot exists");
434        assert!(store.current_id().is_some());
435    }
436}
sift_core/index/store.rs

sift_core/index/
store.rs
