siftdb_core/
lib.rs

1pub mod storage;
2pub mod types;
3pub mod ingest;
4pub mod index;
5pub mod query;
6pub mod bench;
7pub mod fst_index;
8pub mod incremental;
9pub mod inverted_index;
10pub mod tombstone;
11pub mod locking;
12pub mod compaction;
13
14pub use types::*;
15
16use anyhow::Result;
17
18/// SiftDB collection - the main entry point
19pub struct SiftDB {
20    path: std::path::PathBuf,
21}
22
23impl SiftDB {
24    /// Open an existing SiftDB collection
25    pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
26        let path = path.as_ref().to_path_buf();
27        if !path.exists() {
28            anyhow::bail!("SiftDB collection does not exist at: {}", path.display());
29        }
30        Ok(Self { path })
31    }
32
33    /// Initialize a new SiftDB collection
34    pub fn init<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {
35        let path = path.as_ref().to_path_buf();
36        
37        // Create directory structure
38        std::fs::create_dir_all(&path)?;
39        std::fs::create_dir_all(path.join("store"))?;
40        std::fs::create_dir_all(path.join("index"))?;
41        std::fs::create_dir_all(path.join("tmp"))?;
42        std::fs::create_dir_all(path.join("gc"))?;
43
44        // Create initial manifest
45        let manifest = Manifest::new(0);
46        manifest.write_to_file(&path.join("MANIFEST.a"))?;
47
48        Ok(Self { path })
49    }
50
51    /// Get a snapshot of the current state
52    pub fn snapshot(&self) -> Result<Snapshot> {
53        let manifest_path = self.path.join("MANIFEST.a");
54        let manifest = if manifest_path.exists() {
55            Manifest::read_from_file(&manifest_path)?
56        } else {
57            let manifest_path = self.path.join("MANIFEST.b");
58            if manifest_path.exists() {
59                Manifest::read_from_file(&manifest_path)?
60            } else {
61                anyhow::bail!("No valid manifest found");
62            }
63        };
64
65        // Load indexes once when creating snapshot
66        let path_index = crate::index::PathIndex::read_from_file(&self.path.join("index/path.json"))?;
67        let handles_map = crate::index::HandlesMap::read_from_file(&self.path.join("index/handles.json"))?;
68        
69        // Load inverted index (if it exists, otherwise create empty one)
70        let inverted_index = if self.path.join("index/terms.fst").exists() && self.path.join("index/posting_lists.json").exists() {
71            crate::inverted_index::InvertedIndex::load_from_files(
72                &self.path.join("index/terms.fst"),
73                &self.path.join("index/posting_lists.json")
74            )?
75        } else {
76            crate::inverted_index::InvertedIndex::new()
77        };
78
79        Ok(Snapshot {
80            collection_path: self.path.clone(),
81            epoch: manifest.epoch,
82            path_index,
83            handles_map,
84            inverted_index,
85            segment_cache: std::collections::HashMap::new(),
86        })
87    }
88
89    /// Perform incremental update of the collection
90    pub fn incremental_update(
91        &self,
92        source_path: &std::path::Path,
93        includes: &[String],
94        excludes: &[String],
95    ) -> Result<crate::incremental::DeltaManifest> {
96        let updater = crate::incremental::IncrementalUpdater::new(&self.path);
97        
98        // Scan for changes
99        let changes = updater.scan_for_changes(source_path, includes, excludes)?;
100        
101        if changes.is_empty() {
102            anyhow::bail!("No changes detected since last import");
103        }
104
105        println!("Found {} changed files:", changes.len());
106        for change in &changes {
107            match change.change_type {
108                crate::incremental::ChangeType::Added => {
109                    println!("  + {}", change.path.display());
110                }
111                crate::incremental::ChangeType::Modified => {
112                    println!("  M {}", change.path.display());
113                }
114                crate::incremental::ChangeType::Deleted => {
115                    println!("  - {}", change.path.display());
116                }
117            }
118        }
119
120        // Apply changes
121        let delta_manifest = updater.apply_changes(changes, source_path)?;
122        
123        Ok(delta_manifest)
124    }
125
126    /// Check if incremental update is available
127    pub fn has_incremental_cache(&self) -> bool {
128        let cache_path = self.path.join("index").join("file_cache.json");
129        cache_path.exists()
130    }
131}
132
133pub fn add(left: u64, right: u64) -> u64 {
134    left + right
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140
141    #[test]
142    fn it_works() {
143        let result = add(2, 2);
144        assert_eq!(result, 4);
145    }
146}
siftdb_core/lib.rs

siftdb_core/
lib.rs
