sinter_store/
store.rs

1use std::collections::HashMap;
2use std::path::Path;
3
4use redb::{
5    Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable,
6    ReadableTableMetadata, TableDefinition,
7};
8use sinter_core::{Edge, FileFacts, Graph, Node, NodeId, Reference, UnresolvedReference};
9
10use crate::error::StoreError;
11
12pub(crate) const NODES: TableDefinition<&str, &[u8]> = TableDefinition::new("nodes");
13/// Adjacency, keyed by src id. Values are postcard-encoded edges; the
14/// multimap holds parallel edges and dedups byte-identical ones, matching
15/// `Graph` semantics.
16pub(crate) const OUT_EDGES: MultimapTableDefinition<&str, &[u8]> =
17    MultimapTableDefinition::new("out_edges");
18/// Reverse adjacency, keyed by dst id — reverse blast radius reads this.
19pub(crate) const IN_EDGES: MultimapTableDefinition<&str, &[u8]> =
20    MultimapTableDefinition::new("in_edges");
21/// Unresolved references, keyed by file. First-class outcome (R2): stored
22/// and countable, replaced per file on re-resolution.
23pub(crate) const UNRESOLVED: MultimapTableDefinition<&str, &[u8]> =
24    MultimapTableDefinition::new("unresolved");
25/// Per-file extraction truth, content-addressed. Every derived table
26/// (nodes, edges, indexes) rebuilds from here for exactly the changed files.
27pub(crate) const FILE_FACTS: TableDefinition<&str, &[u8]> = TableDefinition::new("file_facts");
28/// file -> content hash, decoded without touching the facts blob.
29pub(crate) const FILE_HASH: TableDefinition<&str, &str> = TableDefinition::new("file_hash");
30/// reference name -> files containing a reference with that name; the
31/// resolution invalidation index.
32pub(crate) const NAME_REFS: MultimapTableDefinition<&str, &str> =
33    MultimapTableDefinition::new("name_refs");
34/// symbol name -> interned node ids, exact-match query index.
35pub(crate) const NAME_NODES: MultimapTableDefinition<&str, u32> =
36    MultimapTableDefinition::new("name_nodes");
37/// lowercased trigram -> interned node ids, fuzzy query index.
38pub(crate) const TRIGRAMS: MultimapTableDefinition<&str, u32> =
39    MultimapTableDefinition::new("trigrams");
40/// lowercased word -> interned node ids: recall index over name subwords,
41/// doc, signature, and path segments (see `search::node_tokens`).
42/// Values are interned (u32) — node-id strings repeated dozens of times
43/// were 58% of stored bytes before interning (bench finding).
44pub(crate) const TOKENS_WORDS: MultimapTableDefinition<&str, u32> =
45    MultimapTableDefinition::new("tokens_words");
46/// Interner: u32 -> node id string and its reverse. Index tables store the
47/// u32; readers translate back on materialization.
48pub(crate) const INTERN: TableDefinition<u32, &str> = TableDefinition::new("intern");
49pub(crate) const INTERN_REV: TableDefinition<&str, u32> = TableDefinition::new("intern_rev");
50/// file -> import references only (compact). Re-export chain walking needs
51/// every file's imports without decoding full facts corpus-wide.
52pub(crate) const IMPORTS: MultimapTableDefinition<&str, &[u8]> =
53    MultimapTableDefinition::new("imports");
54/// Single-row schema stamp; a mismatch on open wipes the database (facts
55/// are derivable, a stale-format db is not worth migrating).
56pub(crate) const META: TableDefinition<&str, u32> = TableDefinition::new("meta");
57/// Fingerprints of non-source resolution inputs (SCIP index, manifest
58/// module roots), keyed by input kind. A change re-resolves the corpus
59/// without re-extracting. Additive table — absent in older dbs, which
60/// makes the first build after upgrade re-resolve once.
61pub(crate) const RESOLVE_META: TableDefinition<&str, &str> = TableDefinition::new("resolve_meta");
62/// Single-row crash-recovery intent: the union of update deltas not yet
63/// followed by a completed resolution pass (see `Store::update_files` /
64/// `Store::clear_pending_delta`). Additive table — absent in older dbs,
65/// read as empty.
66pub(crate) const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending_delta");
67// v9: FileFacts gained declared fields and unresolved rows gained reasons.
68const SCHEMA_VERSION: u32 = 9;
69
70/// Per-file freshness record: content hash plus the stat identity it was
71/// hashed at. On Unix the identity combines modification and change time,
72/// so rewriting bytes while restoring mtime cannot hide an edit. Encoded
73/// in FILE_HASH as `hash|identity_nanos|len`; old mtime-only rows decode
74/// but miss once against the new identity and are refreshed.
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct FileStamp {
77    pub hash: String,
78    pub identity_nanos: u128,
79    pub len: u64,
80}
81
82impl FileStamp {
83    pub(crate) fn encode(&self) -> String {
84        format!("{}|{}|{}", self.hash, self.identity_nanos, self.len)
85    }
86
87    pub(crate) fn decode(value: &str) -> Self {
88        let mut parts = value.split('|');
89        let hash = parts.next().unwrap_or_default().to_string();
90        Self {
91            hash,
92            identity_nanos: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
93            // len 0 never matches: scan only stamps non-empty files.
94            len: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
95        }
96    }
97}
98
99impl Store {
100    /// The schema version this binary writes.
101    pub const CURRENT_SCHEMA: u32 = SCHEMA_VERSION;
102
103    /// Read a database's schema stamp without opening for write and
104    /// without triggering the wipe-on-mismatch in [`Store::create`].
105    pub fn schema_of(path: impl AsRef<Path>) -> Result<Option<u32>, StoreError> {
106        Self::open(path)?.schema()
107    }
108}
109
110/// Persistent graph store. Point queries never load the whole graph.
111pub struct Store {
112    pub(crate) db: Database,
113}
114
115/// redb opens are exclusive, so a query racing a short-lived build (or a
116/// queue of sibling queries — parallel agents fan out dozens) sees
117/// AlreadyOpen. Backoff rides out the queue; a handle held by a
118/// long-lived process still errors after the budget. Windows gets a
119/// longer budget: file locks release lazily after process exit (handle
120/// teardown + AV scans), so a waiter there can outlive 5s of real
121/// contention that unix clears instantly.
122pub(crate) fn open_retrying(
123    path: &Path,
124    open: fn(&Path) -> Result<Database, redb::DatabaseError>,
125) -> Result<Database, redb::DatabaseError> {
126    let budget = std::time::Duration::from_secs(if cfg!(windows) { 20 } else { 5 });
127    let started = std::time::Instant::now();
128    let mut delay = std::time::Duration::from_millis(10);
129    loop {
130        match open(path) {
131            Err(redb::DatabaseError::DatabaseAlreadyOpen) if started.elapsed() < budget => {
132                std::thread::sleep(delay);
133                delay = (delay * 2).min(std::time::Duration::from_millis(200));
134            }
135            other => return other,
136        }
137    }
138}
139
140/// Create (or open) any redb database under sinter's contention policy —
141/// the one named owner of open-retry behavior for auxiliary databases
142/// (workspace link store) that are not the repository [`Store`].
143pub fn create_database(path: &Path) -> Result<Database, StoreError> {
144    Ok(open_retrying(path, |p| Database::create(p))?)
145}
146
147impl Store {
148    /// Create or open the database and ensure all tables exist. An
149    /// existing database with a different schema version is deleted and
150    /// recreated — the next build re-derives everything from source.
151    pub fn create(path: impl AsRef<Path>) -> Result<Self, StoreError> {
152        let path = path.as_ref();
153        if path.exists() {
154            let db = open_retrying(path, |p| Database::open(p))?;
155            let txn = db.begin_read()?;
156            let stored = match txn.open_table(META) {
157                Ok(table) => table.get("schema")?.map(|g| g.value()),
158                Err(redb::TableError::TableDoesNotExist(_)) => None,
159                Err(e) => return Err(e.into()),
160            };
161            // Older schema: wipe and rebuild forward from source. Newer:
162            // refuse — an outdated binary must never destroy a graph it
163            // cannot rebuild equivalently.
164            if let Some(v) = stored
165                && v > SCHEMA_VERSION
166            {
167                return Err(StoreError::NewerSchema {
168                    stored: v,
169                    supported: SCHEMA_VERSION,
170                });
171            }
172            if stored != Some(SCHEMA_VERSION) {
173                drop(txn);
174                drop(db);
175                std::fs::remove_file(path).map_err(StoreError::Reset)?;
176            }
177        }
178        let store = Self {
179            db: open_retrying(path, |p| Database::create(p))?,
180        };
181        let txn = store.db.begin_write()?;
182        {
183            let mut meta = txn.open_table(META)?;
184            meta.insert("schema", SCHEMA_VERSION)?;
185            drop(meta);
186            txn.open_table(NODES)?;
187            txn.open_table(FILE_FACTS)?;
188            txn.open_table(FILE_HASH)?;
189            txn.open_multimap_table(OUT_EDGES)?;
190            txn.open_multimap_table(IN_EDGES)?;
191            txn.open_multimap_table(UNRESOLVED)?;
192            txn.open_multimap_table(NAME_REFS)?;
193            txn.open_multimap_table(NAME_NODES)?;
194            txn.open_multimap_table(TRIGRAMS)?;
195            txn.open_multimap_table(TOKENS_WORDS)?;
196            txn.open_multimap_table(IMPORTS)?;
197            txn.open_table(INTERN)?;
198            txn.open_table(INTERN_REV)?;
199            txn.open_table(RESOLVE_META)?;
200            txn.open_table(PENDING)?;
201        }
202        txn.commit()?;
203        Ok(store)
204    }
205
206    pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
207        Ok(Self {
208            db: open_retrying(path.as_ref(), |p| Database::open(p))?,
209        })
210    }
211
212    /// The schema stamp of this open database, if any.
213    pub fn schema(&self) -> Result<Option<u32>, StoreError> {
214        let txn = self.db.begin_read()?;
215        match txn.open_table(META) {
216            Ok(table) => Ok(table.get("schema")?.map(|g| g.value())),
217            Err(redb::TableError::TableDoesNotExist(_)) => Ok(None),
218            Err(e) => Err(e.into()),
219        }
220    }
221
222    /// Persist a whole graph in one transaction (test/export convenience;
223    /// the incremental path goes through `update_files`).
224    pub fn write_graph(&self, graph: &Graph) -> Result<(), StoreError> {
225        let txn = self.db.begin_write()?;
226        {
227            let mut nodes = txn.open_table(NODES)?;
228            let mut out = txn.open_multimap_table(OUT_EDGES)?;
229            let mut inn = txn.open_multimap_table(IN_EDGES)?;
230            for node in graph.nodes() {
231                nodes.insert(node.id.as_str(), postcard::to_allocvec(node)?.as_slice())?;
232            }
233            for edge in graph.edges() {
234                let bytes = postcard::to_allocvec(edge)?;
235                out.insert(edge.src.as_str(), bytes.as_slice())?;
236                inn.insert(edge.dst.as_str(), bytes.as_slice())?;
237            }
238        }
239        txn.commit()?;
240        Ok(())
241    }
242
243    /// Total stored unresolved references.
244    pub fn unresolved_count(&self) -> Result<u64, StoreError> {
245        let txn = self.db.begin_read()?;
246        let table = match txn.open_multimap_table(UNRESOLVED) {
247            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(0),
248            other => other?,
249        };
250        Ok(table.len()?)
251    }
252
253    /// Every stored unresolved reference — cross-repo boundary resolution
254    /// input (a workspace resolves these against other members' symbols).
255    pub fn all_unresolved(&self) -> Result<Vec<Reference>, StoreError> {
256        Ok(self
257            .all_unresolved_details()?
258            .into_iter()
259            .map(|u| u.reference)
260            .collect())
261    }
262
263    /// Every unresolved outcome including why the graph could not prove a
264    /// target. Query surfaces use this; workspace linking consumes the raw
265    /// references through [`Store::all_unresolved`].
266    pub fn all_unresolved_details(&self) -> Result<Vec<UnresolvedReference>, StoreError> {
267        let txn = self.db.begin_read()?;
268        let table = match txn.open_multimap_table(UNRESOLVED) {
269            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
270            other => other?,
271        };
272        let mut refs = Vec::new();
273        for entry in table.iter()? {
274            let (_, values) = entry?;
275            for guard in values {
276                refs.push(postcard::from_bytes(guard?.value())?);
277            }
278        }
279        Ok(refs)
280    }
281
282    /// Stored unresolved references, optionally narrowed to one file
283    /// and/or a name (final-segment match, same rule as
284    /// [`Store::unresolved_named`]). The `sinter unresolved` listing.
285    pub fn unresolved_refs(
286        &self,
287        file: Option<&str>,
288        name: Option<&str>,
289    ) -> Result<Vec<Reference>, StoreError> {
290        let mut refs = match file {
291            Some(file) => self.references_in(file)?,
292            None => self.all_unresolved()?,
293        };
294        if let Some(name) = name {
295            refs.retain(|r| name_tail_matches(&r.name, name));
296        }
297        Ok(refs)
298    }
299
300    pub fn unresolved_details(
301        &self,
302        file: Option<&str>,
303        name: Option<&str>,
304    ) -> Result<Vec<UnresolvedReference>, StoreError> {
305        let mut refs = match file {
306            Some(file) => self.unresolved_details_in(file)?,
307            None => self.all_unresolved_details()?,
308        };
309        if let Some(name) = name {
310            refs.retain(|u| name_tail_matches(&u.reference.name, name));
311        }
312        Ok(refs)
313    }
314
315    /// Unresolved references recorded for one file.
316    pub fn references_in(&self, file: &str) -> Result<Vec<Reference>, StoreError> {
317        Ok(self
318            .unresolved_details_in(file)?
319            .into_iter()
320            .map(|u| u.reference)
321            .collect())
322    }
323
324    pub fn unresolved_details_in(
325        &self,
326        file: &str,
327    ) -> Result<Vec<UnresolvedReference>, StoreError> {
328        let txn = self.db.begin_read()?;
329        let table = match txn.open_multimap_table(UNRESOLVED) {
330            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
331            other => other?,
332        };
333        let mut refs = Vec::new();
334        for guard in table.get(file)? {
335            refs.push(postcard::from_bytes(guard?.value())?);
336        }
337        Ok(refs)
338    }
339
340    /// The fingerprint a non-source resolution input (key: "scip",
341    /// "module_roots") was last resolved against, if any.
342    pub fn resolve_fingerprint(&self, key: &str) -> Result<Option<String>, StoreError> {
343        let txn = self.db.begin_read()?;
344        let table = match txn.open_table(RESOLVE_META) {
345            Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
346            other => other?,
347        };
348        Ok(table.get(key)?.map(|g| g.value().to_string()))
349    }
350
351    /// Idempotent: an unchanged fingerprint opens no write transaction,
352    /// keeping a clean build write-free (parallel readers never queue
353    /// behind redb's exclusive writer for a no-op).
354    pub fn set_resolve_fingerprint(
355        &self,
356        key: &str,
357        fingerprint: Option<&str>,
358    ) -> Result<(), StoreError> {
359        if self.resolve_fingerprint(key)?.as_deref() == fingerprint {
360            return Ok(());
361        }
362        let txn = self.db.begin_write()?;
363        {
364            let mut table = txn.open_table(RESOLVE_META)?;
365            match fingerprint {
366                Some(f) => {
367                    table.insert(key, f)?;
368                }
369                None => {
370                    table.remove(key)?;
371                }
372            }
373        }
374        txn.commit()?;
375        Ok(())
376    }
377
378    /// Unresolved references whose written name ends in this name — the
379    /// honest-empty signal for blast-radius queries: a nonzero count means
380    /// the graph may be missing dependents of a symbol with that name.
381    pub fn unresolved_named(&self, name: &str) -> Result<usize, StoreError> {
382        let files = self.ref_files(&std::collections::BTreeSet::from([name.to_string()]))?;
383        let mut count = 0;
384        for file in files {
385            count += self
386                .references_in(&file)?
387                .iter()
388                .filter(|r| name_tail_matches(&r.name, name))
389                .count();
390        }
391        Ok(count)
392    }
393
394    pub fn node(&self, id: &NodeId) -> Result<Option<Node>, StoreError> {
395        let txn = self.db.begin_read()?;
396        let table = txn.open_table(NODES)?;
397        match table.get(id.as_str())? {
398            Some(guard) => Ok(Some(postcard::from_bytes(guard.value())?)),
399            None => Ok(None),
400        }
401    }
402
403    pub fn node_count(&self) -> Result<u64, StoreError> {
404        let txn = self.db.begin_read()?;
405        Ok(txn.open_table(NODES)?.len()?)
406    }
407
408    pub fn edge_count(&self) -> Result<u64, StoreError> {
409        let txn = self.db.begin_read()?;
410        Ok(txn.open_multimap_table(OUT_EDGES)?.len()?)
411    }
412
413    /// Edges leaving `id`.
414    pub fn out_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
415        self.adjacent(OUT_EDGES, id)
416    }
417
418    /// Edges arriving at `id`.
419    pub fn in_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
420        self.adjacent(IN_EDGES, id)
421    }
422
423    /// Incoming edges for several nodes under one read transaction. Query
424    /// ranking uses this instead of opening one redb snapshot per candidate.
425    pub fn in_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
426        let txn = self.db.begin_read()?;
427        let table = txn.open_multimap_table(IN_EDGES)?;
428        let mut found = HashMap::with_capacity(ids.len());
429        for id in ids {
430            let mut edges = Vec::new();
431            for guard in table.get(id.as_str())? {
432                edges.push(postcard::from_bytes(guard?.value())?);
433            }
434            found.insert(id.clone(), edges);
435        }
436        Ok(found)
437    }
438
439    fn adjacent(
440        &self,
441        table: MultimapTableDefinition<&str, &[u8]>,
442        id: &NodeId,
443    ) -> Result<Vec<Edge>, StoreError> {
444        let txn = self.db.begin_read()?;
445        let table = txn.open_multimap_table(table)?;
446        let mut edges = Vec::new();
447        for guard in table.get(id.as_str())? {
448            edges.push(postcard::from_bytes(guard?.value())?);
449        }
450        Ok(edges)
451    }
452
453    /// (file, stamp) for every stored file — the changed-set diff base.
454    pub fn file_hashes(&self) -> Result<Vec<(String, FileStamp)>, StoreError> {
455        let txn = self.db.begin_read()?;
456        let table = txn.open_table(FILE_HASH)?;
457        let mut out = Vec::new();
458        for entry in table.iter()? {
459            let (k, v) = entry?;
460            out.push((k.value().to_string(), FileStamp::decode(v.value())));
461        }
462        Ok(out)
463    }
464
465    pub fn facts(&self, file: &str) -> Result<Option<FileFacts>, StoreError> {
466        let txn = self.db.begin_read()?;
467        let table = txn.open_table(FILE_FACTS)?;
468        match table.get(file)? {
469            Some(guard) => Ok(Some(crate::update::decode_facts(guard.value())?)),
470            None => Ok(None),
471        }
472    }
473
474    /// Files whose most recently extracted syntax tree contained errors.
475    /// Coverage reporting uses the complete persisted set, not only files
476    /// changed by the latest incremental pass.
477    pub fn syntax_error_files(&self) -> Result<Vec<String>, StoreError> {
478        let txn = self.db.begin_read()?;
479        let table = txn.open_table(FILE_FACTS)?;
480        let mut files = Vec::new();
481        for entry in table.iter()? {
482            let (file, bytes) = entry?;
483            let facts = crate::update::decode_facts(bytes.value())?;
484            if facts.has_syntax_errors {
485                files.push(file.value().to_string());
486            }
487        }
488        files.sort();
489        Ok(files)
490    }
491
492    /// Reclaim free pages. Worth running after bulk rebuilds; skipped on
493    /// incremental updates (it rewrites the file and would blow the <1s
494    /// one-file-edit budget). redb compaction is iterative — repeat until
495    /// it reports no further progress (bounded).
496    pub fn compact(&mut self) -> Result<bool, StoreError> {
497        let mut any = false;
498        for _ in 0..16 {
499            if !self.db.compact()? {
500                break;
501            }
502            any = true;
503        }
504        Ok(any)
505    }
506
507    /// Every stored import reference — re-export chain-walking input.
508    pub fn all_imports(&self) -> Result<Vec<Reference>, StoreError> {
509        let txn = self.db.begin_read()?;
510        let table = txn.open_multimap_table(IMPORTS)?;
511        let mut refs = Vec::new();
512        for entry in table.iter()? {
513            let (_, values) = entry?;
514            for guard in values {
515                refs.push(postcard::from_bytes(guard?.value())?);
516            }
517        }
518        Ok(refs)
519    }
520
521    /// Every stored node — resolution index input. Compact scan of the node
522    /// table; queries never need this.
523    pub fn all_nodes(&self) -> Result<Vec<Node>, StoreError> {
524        let txn = self.db.begin_read()?;
525        let table = txn.open_table(NODES)?;
526        let mut nodes = Vec::new();
527        for entry in table.iter()? {
528            nodes.push(postcard::from_bytes(entry?.1.value())?);
529        }
530        Ok(nodes)
531    }
532
533    /// Non-`Contains` in-degree per node id, streamed straight off the
534    /// IN_EDGES table — hub ranking without materializing (and
535    /// re-validating) the whole graph. Nodes with zero such in-edges are
536    /// omitted.
537    pub fn in_degrees(&self) -> Result<Vec<(String, usize)>, StoreError> {
538        let txn = self.db.begin_read()?;
539        let table = txn.open_multimap_table(IN_EDGES)?;
540        let mut out = Vec::new();
541        for entry in table.iter()? {
542            let (key, values) = entry?;
543            let mut n = 0usize;
544            for guard in values {
545                let edge: Edge = postcard::from_bytes(guard?.value())?;
546                if edge.relation != sinter_core::Relation::Contains {
547                    n += 1;
548                }
549            }
550            if n > 0 {
551                out.push((key.value().to_string(), n));
552            }
553        }
554        Ok(out)
555    }
556
557    /// Rebuild the full in-memory graph, re-validating every invariant.
558    /// Export/debug path only — queries must not need this.
559    pub fn read_graph(&self) -> Result<Graph, StoreError> {
560        let txn = self.db.begin_read()?;
561        let mut graph = Graph::new();
562        {
563            let nodes = txn.open_table(NODES)?;
564            for entry in nodes.iter()? {
565                let (_, value) = entry?;
566                graph.add_node(postcard::from_bytes(value.value())?)?;
567            }
568        }
569        {
570            let out = txn.open_multimap_table(OUT_EDGES)?;
571            for entry in out.iter()? {
572                let (_, values) = entry?;
573                for guard in values {
574                    graph.add_edge(postcard::from_bytes(guard?.value())?)?;
575                }
576            }
577        }
578        Ok(graph)
579    }
580}
581
582/// Does a written reference name (`acme_common::connect_grpc_channel`,
583/// `pkg.Func`) end at exactly this name?
584fn name_tail_matches(written: &str, name: &str) -> bool {
585    // Exact final-segment equality across every language's separators —
586    // boundary substring matching over-counted short common names
587    // (`run`, `install`) into the honest-empty note.
588    let tail = written.rsplit("::").next().unwrap_or(written);
589    let tail = tail.rsplit(['/', '.']).next().unwrap_or(tail);
590    tail == name
591}
sinter_store/store.rs

sinter_store/
store.rs
