sinter_store/
store.rs

1use std::path::Path;
2
3use redb::{
4    Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable,
5    ReadableTableMetadata, TableDefinition,
6};
7use sinter_core::{Edge, FileFacts, Graph, Node, NodeId, Reference};
8
9use crate::error::StoreError;
10
11pub(crate) const NODES: TableDefinition<&str, &[u8]> = TableDefinition::new("nodes");
12/// Adjacency, keyed by src id. Values are postcard-encoded edges; the
13/// multimap holds parallel edges and dedups byte-identical ones, matching
14/// `Graph` semantics.
15pub(crate) const OUT_EDGES: MultimapTableDefinition<&str, &[u8]> =
16    MultimapTableDefinition::new("out_edges");
17/// Reverse adjacency, keyed by dst id — reverse blast radius reads this.
18pub(crate) const IN_EDGES: MultimapTableDefinition<&str, &[u8]> =
19    MultimapTableDefinition::new("in_edges");
20/// Unresolved references, keyed by file. First-class outcome (R2): stored
21/// and countable, replaced per file on re-resolution.
22pub(crate) const UNRESOLVED: MultimapTableDefinition<&str, &[u8]> =
23    MultimapTableDefinition::new("unresolved");
24/// Per-file extraction truth, content-addressed. Every derived table
25/// (nodes, edges, indexes) rebuilds from here for exactly the changed files.
26pub(crate) const FILE_FACTS: TableDefinition<&str, &[u8]> = TableDefinition::new("file_facts");
27/// file -> content hash, decoded without touching the facts blob.
28pub(crate) const FILE_HASH: TableDefinition<&str, &str> = TableDefinition::new("file_hash");
29/// reference name -> files containing a reference with that name; the
30/// resolution invalidation index.
31pub(crate) const NAME_REFS: MultimapTableDefinition<&str, &str> =
32    MultimapTableDefinition::new("name_refs");
33/// symbol name -> interned node ids, exact-match query index.
34pub(crate) const NAME_NODES: MultimapTableDefinition<&str, u32> =
35    MultimapTableDefinition::new("name_nodes");
36/// lowercased trigram -> interned node ids, fuzzy query index.
37pub(crate) const TRIGRAMS: MultimapTableDefinition<&str, u32> =
38    MultimapTableDefinition::new("trigrams");
39/// lowercased word -> interned node ids: recall index over name subwords,
40/// doc, signature, and path segments (see `search::node_tokens`).
41/// Values are interned (u32) — node-id strings repeated dozens of times
42/// were 58% of stored bytes before interning (bench finding).
43pub(crate) const TOKENS_WORDS: MultimapTableDefinition<&str, u32> =
44    MultimapTableDefinition::new("tokens_words");
45/// Interner: u32 -> node id string and its reverse. Index tables store the
46/// u32; readers translate back on materialization.
47pub(crate) const INTERN: TableDefinition<u32, &str> = TableDefinition::new("intern");
48pub(crate) const INTERN_REV: TableDefinition<&str, u32> = TableDefinition::new("intern_rev");
49/// file -> import references only (compact). Re-export chain walking needs
50/// every file's imports without decoding full facts corpus-wide.
51pub(crate) const IMPORTS: MultimapTableDefinition<&str, &[u8]> =
52    MultimapTableDefinition::new("imports");
53/// Single-row schema stamp; a mismatch on open wipes the database (facts
54/// are derivable, a stale-format db is not worth migrating).
55pub(crate) const META: TableDefinition<&str, u32> = TableDefinition::new("meta");
56/// Fingerprints of non-source resolution inputs (SCIP index, manifest
57/// module roots), keyed by input kind. A change re-resolves the corpus
58/// without re-extracting. Additive table — absent in older dbs, which
59/// makes the first build after upgrade re-resolve once.
60pub(crate) const RESOLVE_META: TableDefinition<&str, &str> = TableDefinition::new("resolve_meta");
61/// Single-row crash-recovery intent: the union of update deltas not yet
62/// followed by a completed resolution pass (see `Store::update_files` /
63/// `Store::clear_pending_delta`). Additive table — absent in older dbs,
64/// read as empty.
65pub(crate) const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending_delta");
66// v8: Edge gained `site` (call-site span); postcard wire format changed.
67const SCHEMA_VERSION: u32 = 8;
68
69/// Per-file freshness record: content hash plus the stat identity
70/// (mtime, len) it was hashed at. A scan whose stat matches reuses the
71/// hash without reading the file (the `make` trick). Encoded in
72/// FILE_HASH as `hash|mtime_nanos|len`; a bare hash (no stamp) never
73/// matches a stat, forcing a re-hash.
74#[derive(Debug, Clone, PartialEq, Eq)]
75pub struct FileStamp {
76    pub hash: String,
77    pub mtime_nanos: u128,
78    pub len: u64,
79}
80
81impl FileStamp {
82    pub(crate) fn encode(&self) -> String {
83        format!("{}|{}|{}", self.hash, self.mtime_nanos, self.len)
84    }
85
86    pub(crate) fn decode(value: &str) -> Self {
87        let mut parts = value.split('|');
88        let hash = parts.next().unwrap_or_default().to_string();
89        Self {
90            hash,
91            mtime_nanos: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
92            // len 0 never matches: scan only stamps non-empty files.
93            len: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
94        }
95    }
96}
97
98impl Store {
99    /// The schema version this binary writes.
100    pub const CURRENT_SCHEMA: u32 = SCHEMA_VERSION;
101
102    /// Read a database's schema stamp without opening for write and
103    /// without triggering the wipe-on-mismatch in [`Store::create`].
104    pub fn schema_of(path: impl AsRef<Path>) -> Result<Option<u32>, StoreError> {
105        Self::open(path)?.schema()
106    }
107}
108
109/// Persistent graph store. Point queries never load the whole graph.
110pub struct Store {
111    pub(crate) db: Database,
112}
113
114/// redb opens are exclusive, so a query racing a short-lived build (or a
115/// queue of sibling queries — parallel agents fan out dozens) sees
116/// AlreadyOpen. Backoff rides out the queue; a handle held by a
117/// long-lived process still errors after the budget. Windows gets a
118/// longer budget: file locks release lazily after process exit (handle
119/// teardown + AV scans), so a waiter there can outlive 5s of real
120/// contention that unix clears instantly.
121pub(crate) fn open_retrying(
122    path: &Path,
123    open: fn(&Path) -> Result<Database, redb::DatabaseError>,
124) -> Result<Database, redb::DatabaseError> {
125    let budget = std::time::Duration::from_secs(if cfg!(windows) { 20 } else { 5 });
126    let started = std::time::Instant::now();
127    let mut delay = std::time::Duration::from_millis(10);
128    loop {
129        match open(path) {
130            Err(redb::DatabaseError::DatabaseAlreadyOpen) if started.elapsed() < budget => {
131                std::thread::sleep(delay);
132                delay = (delay * 2).min(std::time::Duration::from_millis(200));
133            }
134            other => return other,
135        }
136    }
137}
138
139/// Create (or open) any redb database under sinter's contention policy —
140/// the one named owner of open-retry behavior for auxiliary databases
141/// (workspace link store) that are not the repository [`Store`].
142pub fn create_database(path: &Path) -> Result<Database, StoreError> {
143    Ok(open_retrying(path, |p| Database::create(p))?)
144}
145
146impl Store {
147    /// Create or open the database and ensure all tables exist. An
148    /// existing database with a different schema version is deleted and
149    /// recreated — the next build re-derives everything from source.
150    pub fn create(path: impl AsRef<Path>) -> Result<Self, StoreError> {
151        let path = path.as_ref();
152        if path.exists() {
153            let db = open_retrying(path, |p| Database::open(p))?;
154            let txn = db.begin_read()?;
155            let stored = match txn.open_table(META) {
156                Ok(table) => table.get("schema")?.map(|g| g.value()),
157                Err(redb::TableError::TableDoesNotExist(_)) => None,
158                Err(e) => return Err(e.into()),
159            };
160            // Older schema: wipe and rebuild forward from source. Newer:
161            // refuse — an outdated binary must never destroy a graph it
162            // cannot rebuild equivalently.
163            if let Some(v) = stored
164                && v > SCHEMA_VERSION
165            {
166                return Err(StoreError::NewerSchema {
167                    stored: v,
168                    supported: SCHEMA_VERSION,
169                });
170            }
171            if stored != Some(SCHEMA_VERSION) {
172                drop(txn);
173                drop(db);
174                std::fs::remove_file(path).map_err(StoreError::Reset)?;
175            }
176        }
177        let store = Self {
178            db: open_retrying(path, |p| Database::create(p))?,
179        };
180        let txn = store.db.begin_write()?;
181        {
182            let mut meta = txn.open_table(META)?;
183            meta.insert("schema", SCHEMA_VERSION)?;
184            drop(meta);
185            txn.open_table(NODES)?;
186            txn.open_table(FILE_FACTS)?;
187            txn.open_table(FILE_HASH)?;
188            txn.open_multimap_table(OUT_EDGES)?;
189            txn.open_multimap_table(IN_EDGES)?;
190            txn.open_multimap_table(UNRESOLVED)?;
191            txn.open_multimap_table(NAME_REFS)?;
192            txn.open_multimap_table(NAME_NODES)?;
193            txn.open_multimap_table(TRIGRAMS)?;
194            txn.open_multimap_table(TOKENS_WORDS)?;
195            txn.open_multimap_table(IMPORTS)?;
196            txn.open_table(INTERN)?;
197            txn.open_table(INTERN_REV)?;
198            txn.open_table(RESOLVE_META)?;
199            txn.open_table(PENDING)?;
200        }
201        txn.commit()?;
202        Ok(store)
203    }
204
205    pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
206        Ok(Self {
207            db: open_retrying(path.as_ref(), |p| Database::open(p))?,
208        })
209    }
210
211    /// The schema stamp of this open database, if any.
212    pub fn schema(&self) -> Result<Option<u32>, StoreError> {
213        let txn = self.db.begin_read()?;
214        match txn.open_table(META) {
215            Ok(table) => Ok(table.get("schema")?.map(|g| g.value())),
216            Err(redb::TableError::TableDoesNotExist(_)) => Ok(None),
217            Err(e) => Err(e.into()),
218        }
219    }
220
221    /// Persist a whole graph in one transaction (test/export convenience;
222    /// the incremental path goes through `update_files`).
223    pub fn write_graph(&self, graph: &Graph) -> Result<(), StoreError> {
224        let txn = self.db.begin_write()?;
225        {
226            let mut nodes = txn.open_table(NODES)?;
227            let mut out = txn.open_multimap_table(OUT_EDGES)?;
228            let mut inn = txn.open_multimap_table(IN_EDGES)?;
229            for node in graph.nodes() {
230                nodes.insert(node.id.as_str(), postcard::to_allocvec(node)?.as_slice())?;
231            }
232            for edge in graph.edges() {
233                let bytes = postcard::to_allocvec(edge)?;
234                out.insert(edge.src.as_str(), bytes.as_slice())?;
235                inn.insert(edge.dst.as_str(), bytes.as_slice())?;
236            }
237        }
238        txn.commit()?;
239        Ok(())
240    }
241
242    /// Total stored unresolved references.
243    pub fn unresolved_count(&self) -> Result<u64, StoreError> {
244        let txn = self.db.begin_read()?;
245        let table = match txn.open_multimap_table(UNRESOLVED) {
246            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(0),
247            other => other?,
248        };
249        Ok(table.len()?)
250    }
251
252    /// Every stored unresolved reference — cross-repo boundary resolution
253    /// input (a workspace resolves these against other members' symbols).
254    pub fn all_unresolved(&self) -> Result<Vec<Reference>, StoreError> {
255        let txn = self.db.begin_read()?;
256        let table = match txn.open_multimap_table(UNRESOLVED) {
257            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
258            other => other?,
259        };
260        let mut refs = Vec::new();
261        for entry in table.iter()? {
262            let (_, values) = entry?;
263            for guard in values {
264                refs.push(postcard::from_bytes(guard?.value())?);
265            }
266        }
267        Ok(refs)
268    }
269
270    /// Stored unresolved references, optionally narrowed to one file
271    /// and/or a name (final-segment match, same rule as
272    /// [`Store::unresolved_named`]). The `sinter unresolved` listing.
273    pub fn unresolved_refs(
274        &self,
275        file: Option<&str>,
276        name: Option<&str>,
277    ) -> Result<Vec<Reference>, StoreError> {
278        let mut refs = match file {
279            Some(file) => self.references_in(file)?,
280            None => self.all_unresolved()?,
281        };
282        if let Some(name) = name {
283            refs.retain(|r| name_tail_matches(&r.name, name));
284        }
285        Ok(refs)
286    }
287
288    /// Unresolved references recorded for one file.
289    pub fn references_in(&self, file: &str) -> Result<Vec<Reference>, StoreError> {
290        let txn = self.db.begin_read()?;
291        let table = match txn.open_multimap_table(UNRESOLVED) {
292            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
293            other => other?,
294        };
295        let mut refs = Vec::new();
296        for guard in table.get(file)? {
297            refs.push(postcard::from_bytes(guard?.value())?);
298        }
299        Ok(refs)
300    }
301
302    /// The fingerprint a non-source resolution input (key: "scip",
303    /// "module_roots") was last resolved against, if any.
304    pub fn resolve_fingerprint(&self, key: &str) -> Result<Option<String>, StoreError> {
305        let txn = self.db.begin_read()?;
306        let table = match txn.open_table(RESOLVE_META) {
307            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
308            other => other?,
309        };
310        Ok(table.get(key)?.map(|g| g.value().to_string()))
311    }
312
313    /// Idempotent: an unchanged fingerprint opens no write transaction,
314    /// keeping a clean build write-free (parallel readers never queue
315    /// behind redb's exclusive writer for a no-op).
316    pub fn set_resolve_fingerprint(
317        &self,
318        key: &str,
319        fingerprint: Option<&str>,
320    ) -> Result<(), StoreError> {
321        if self.resolve_fingerprint(key)?.as_deref() == fingerprint {
322            return Ok(());
323        }
324        let txn = self.db.begin_write()?;
325        {
326            let mut table = txn.open_table(RESOLVE_META)?;
327            match fingerprint {
328                Some(f) => {
329                    table.insert(key, f)?;
330                }
331                None => {
332                    table.remove(key)?;
333                }
334            }
335        }
336        txn.commit()?;
337        Ok(())
338    }
339
340    /// Unresolved references whose written name ends in this name — the
341    /// honest-empty signal for blast-radius queries: a nonzero count means
342    /// the graph may be missing dependents of a symbol with that name.
343    pub fn unresolved_named(&self, name: &str) -> Result<usize, StoreError> {
344        let files = self.ref_files(&std::collections::BTreeSet::from([name.to_string()]))?;
345        let mut count = 0;
346        for file in files {
347            count += self
348                .references_in(&file)?
349                .iter()
350                .filter(|r| name_tail_matches(&r.name, name))
351                .count();
352        }
353        Ok(count)
354    }
355
356    pub fn node(&self, id: &NodeId) -> Result<Option<Node>, StoreError> {
357        let txn = self.db.begin_read()?;
358        let table = txn.open_table(NODES)?;
359        match table.get(id.as_str())? {
360            Some(guard) => Ok(Some(postcard::from_bytes(guard.value())?)),
361            None => Ok(None),
362        }
363    }
364
365    pub fn node_count(&self) -> Result<u64, StoreError> {
366        let txn = self.db.begin_read()?;
367        Ok(txn.open_table(NODES)?.len()?)
368    }
369
370    pub fn edge_count(&self) -> Result<u64, StoreError> {
371        let txn = self.db.begin_read()?;
372        Ok(txn.open_multimap_table(OUT_EDGES)?.len()?)
373    }
374
375    /// Edges leaving `id`.
376    pub fn out_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
377        self.adjacent(OUT_EDGES, id)
378    }
379
380    /// Edges arriving at `id`.
381    pub fn in_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
382        self.adjacent(IN_EDGES, id)
383    }
384
385    fn adjacent(
386        &self,
387        table: MultimapTableDefinition<&str, &[u8]>,
388        id: &NodeId,
389    ) -> Result<Vec<Edge>, StoreError> {
390        let txn = self.db.begin_read()?;
391        let table = txn.open_multimap_table(table)?;
392        let mut edges = Vec::new();
393        for guard in table.get(id.as_str())? {
394            edges.push(postcard::from_bytes(guard?.value())?);
395        }
396        Ok(edges)
397    }
398
399    /// (file, stamp) for every stored file — the changed-set diff base.
400    pub fn file_hashes(&self) -> Result<Vec<(String, FileStamp)>, StoreError> {
401        let txn = self.db.begin_read()?;
402        let table = txn.open_table(FILE_HASH)?;
403        let mut out = Vec::new();
404        for entry in table.iter()? {
405            let (k, v) = entry?;
406            out.push((k.value().to_string(), FileStamp::decode(v.value())));
407        }
408        Ok(out)
409    }
410
411    pub fn facts(&self, file: &str) -> Result<Option<FileFacts>, StoreError> {
412        let txn = self.db.begin_read()?;
413        let table = txn.open_table(FILE_FACTS)?;
414        match table.get(file)? {
415            Some(guard) => Ok(Some(crate::update::decode_facts(guard.value())?)),
416            None => Ok(None),
417        }
418    }
419
420    /// Reclaim free pages. Worth running after bulk rebuilds; skipped on
421    /// incremental updates (it rewrites the file and would blow the <1s
422    /// one-file-edit budget). redb compaction is iterative — repeat until
423    /// it reports no further progress (bounded).
424    pub fn compact(&mut self) -> Result<bool, StoreError> {
425        let mut any = false;
426        for _ in 0..16 {
427            if !self.db.compact()? {
428                break;
429            }
430            any = true;
431        }
432        Ok(any)
433    }
434
435    /// Every stored import reference — re-export chain-walking input.
436    pub fn all_imports(&self) -> Result<Vec<Reference>, StoreError> {
437        let txn = self.db.begin_read()?;
438        let table = txn.open_multimap_table(IMPORTS)?;
439        let mut refs = Vec::new();
440        for entry in table.iter()? {
441            let (_, values) = entry?;
442            for guard in values {
443                refs.push(postcard::from_bytes(guard?.value())?);
444            }
445        }
446        Ok(refs)
447    }
448
449    /// Every stored node — resolution index input. Compact scan of the node
450    /// table; queries never need this.
451    pub fn all_nodes(&self) -> Result<Vec<Node>, StoreError> {
452        let txn = self.db.begin_read()?;
453        let table = txn.open_table(NODES)?;
454        let mut nodes = Vec::new();
455        for entry in table.iter()? {
456            nodes.push(postcard::from_bytes(entry?.1.value())?);
457        }
458        Ok(nodes)
459    }
460
461    /// Non-`Contains` in-degree per node id, streamed straight off the
462    /// IN_EDGES table — hub ranking without materializing (and
463    /// re-validating) the whole graph. Nodes with zero such in-edges are
464    /// omitted.
465    pub fn in_degrees(&self) -> Result<Vec<(String, usize)>, StoreError> {
466        let txn = self.db.begin_read()?;
467        let table = txn.open_multimap_table(IN_EDGES)?;
468        let mut out = Vec::new();
469        for entry in table.iter()? {
470            let (key, values) = entry?;
471            let mut n = 0usize;
472            for guard in values {
473                let edge: Edge = postcard::from_bytes(guard?.value())?;
474                if edge.relation != sinter_core::Relation::Contains {
475                    n += 1;
476                }
477            }
478            if n > 0 {
479                out.push((key.value().to_string(), n));
480            }
481        }
482        Ok(out)
483    }
484
485    /// Rebuild the full in-memory graph, re-validating every invariant.
486    /// Export/debug path only — queries must not need this.
487    pub fn read_graph(&self) -> Result<Graph, StoreError> {
488        let txn = self.db.begin_read()?;
489        let mut graph = Graph::new();
490        {
491            let nodes = txn.open_table(NODES)?;
492            for entry in nodes.iter()? {
493                let (_, value) = entry?;
494                graph.add_node(postcard::from_bytes(value.value())?)?;
495            }
496        }
497        {
498            let out = txn.open_multimap_table(OUT_EDGES)?;
499            for entry in out.iter()? {
500                let (_, values) = entry?;
501                for guard in values {
502                    graph.add_edge(postcard::from_bytes(guard?.value())?)?;
503                }
504            }
505        }
506        Ok(graph)
507    }
508}
509
510/// Does a written reference name (`sondera_common::connect_grpc_channel`,
511/// `pkg.Func`) end at exactly this name?
512fn name_tail_matches(written: &str, name: &str) -> bool {
513    // Exact final-segment equality across every language's separators —
514    // boundary substring matching over-counted short common names
515    // (`run`, `install`) into the honest-empty note.
516    let tail = written.rsplit("::").next().unwrap_or(written);
517    let tail = tail.rsplit(['/', '.']).next().unwrap_or(tail);
518    tail == name
519}
sinter_store/store.rs

sinter_store/
store.rs
