1use std::collections::HashMap;
2use std::path::Path;
3
4use redb::{
5 Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable,
6 ReadableTableMetadata, TableDefinition,
7};
8use sinter_core::{
9 CorpusScope, Edge, FileFacts, Graph, Node, NodeId, Reference, UnresolvedReference,
10};
11
12use crate::error::StoreError;
13
14pub(crate) const NODES: TableDefinition<&str, &[u8]> = TableDefinition::new("nodes");
15pub(crate) const OUT_EDGES: MultimapTableDefinition<&str, &[u8]> =
19 MultimapTableDefinition::new("out_edges");
20pub(crate) const IN_EDGES: MultimapTableDefinition<&str, &[u8]> =
22 MultimapTableDefinition::new("in_edges");
23pub(crate) const UNRESOLVED: MultimapTableDefinition<&str, &[u8]> =
26 MultimapTableDefinition::new("unresolved");
27pub(crate) const FILE_FACTS: TableDefinition<&str, &[u8]> = TableDefinition::new("file_facts");
30pub(crate) const FILE_HASH: TableDefinition<&str, &str> = TableDefinition::new("file_hash");
32pub(crate) const FILE_SCOPE: TableDefinition<&str, &str> = TableDefinition::new("file_scope");
35pub(crate) const NODE_SCOPE: TableDefinition<&str, &str> = TableDefinition::new("node_scope");
38pub(crate) const NAME_REFS: MultimapTableDefinition<&str, &str> =
41 MultimapTableDefinition::new("name_refs");
42pub(crate) const NAME_NODES: MultimapTableDefinition<&str, u32> =
44 MultimapTableDefinition::new("name_nodes");
45pub(crate) const TRIGRAMS: MultimapTableDefinition<&str, u32> =
47 MultimapTableDefinition::new("trigrams");
48pub(crate) const TOKENS_WORDS: MultimapTableDefinition<&str, u32> =
53 MultimapTableDefinition::new("tokens_words");
54pub(crate) const BODY_TERMS: MultimapTableDefinition<&str, u32> =
58 MultimapTableDefinition::new("body_terms");
59pub(crate) const INTERN: TableDefinition<u32, &str> = TableDefinition::new("intern");
62pub(crate) const INTERN_REV: TableDefinition<&str, u32> = TableDefinition::new("intern_rev");
63pub(crate) const IMPORTS: MultimapTableDefinition<&str, &[u8]> =
66 MultimapTableDefinition::new("imports");
67pub(crate) const META: TableDefinition<&str, u32> = TableDefinition::new("meta");
70pub(crate) const RESOLVE_META: TableDefinition<&str, &str> = TableDefinition::new("resolve_meta");
75pub(crate) const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending_delta");
80const SCHEMA_VERSION: u32 = 13;
87
88#[derive(Debug, Clone, PartialEq, Eq)]
94pub struct FileStamp {
95 pub hash: String,
96 pub identity_nanos: u128,
97 pub len: u64,
98}
99
100impl FileStamp {
101 pub(crate) fn encode(&self) -> String {
102 format!("{}|{}|{}", self.hash, self.identity_nanos, self.len)
103 }
104
105 pub(crate) fn decode(value: &str) -> Self {
106 let mut parts = value.split('|');
107 let hash = parts.next().unwrap_or_default().to_string();
108 Self {
109 hash,
110 identity_nanos: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
111 len: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
113 }
114 }
115}
116
117impl Store {
118 pub const CURRENT_SCHEMA: u32 = SCHEMA_VERSION;
120
121 pub fn schema_of(path: impl AsRef<Path>) -> Result<Option<u32>, StoreError> {
124 let path = path.as_ref();
125 match Self::open_read_only(path) {
126 Ok(store) => store.schema(),
127 Err(_) => Self::open(path)?.schema(),
129 }
130 }
131}
132
133pub struct Store {
135 pub(crate) db: Db,
136}
137
138pub(crate) enum Db {
144 Writable(Database),
145 ReadOnly(redb::ReadOnlyDatabase),
146}
147
148impl Db {
149 pub(crate) fn begin_read(&self) -> Result<redb::ReadTransaction, redb::TransactionError> {
150 match self {
151 Self::Writable(db) => db.begin_read(),
152 Self::ReadOnly(db) => db.begin_read(),
153 }
154 }
155
156 pub(crate) fn begin_write(&self) -> Result<redb::WriteTransaction, StoreError> {
157 match self {
158 Self::Writable(db) => Ok(db.begin_write()?),
159 Self::ReadOnly(_) => Err(StoreError::ReadOnly),
160 }
161 }
162
163 fn compact(&mut self) -> Result<bool, StoreError> {
164 match self {
165 Self::Writable(db) => Ok(db.compact()?),
166 Self::ReadOnly(_) => Err(StoreError::ReadOnly),
167 }
168 }
169}
170
171pub(crate) fn open_retrying<D>(
177 path: &Path,
178 open: fn(&Path) -> Result<D, redb::DatabaseError>,
179) -> Result<D, redb::DatabaseError> {
180 let budget = std::time::Duration::from_secs(120);
183 let notice_after = std::time::Duration::from_secs(1);
184 let started = std::time::Instant::now();
185 let mut delay = std::time::Duration::from_millis(10);
186 let mut noticed = false;
187 loop {
188 match open(path) {
189 Err(redb::DatabaseError::DatabaseAlreadyOpen) if started.elapsed() < budget => {
190 if !noticed && started.elapsed() >= notice_after {
191 noticed = true;
192 eprintln!(
193 "sinter: waiting for another sinter process holding {} (a build in progress?)",
194 path.display()
195 );
196 }
197 std::thread::sleep(delay);
198 delay = (delay * 2).min(std::time::Duration::from_millis(200));
199 }
200 other => return other,
201 }
202 }
203}
204
205pub fn create_database(path: &Path) -> Result<Database, StoreError> {
209 Ok(open_retrying(path, |p| Database::create(p))?)
210}
211
212impl Store {
213 pub fn create(path: impl AsRef<Path>) -> Result<Self, StoreError> {
217 let path = path.as_ref();
218 if path.exists() {
219 let db = open_retrying(path, |p| Database::open(p))?;
220 let txn = db.begin_read()?;
221 let stored = match txn.open_table(META) {
222 Ok(table) => table.get("schema")?.map(|g| g.value()),
223 Err(redb::TableError::TableDoesNotExist(_)) => None,
224 Err(e) => return Err(e.into()),
225 };
226 if let Some(v) = stored
230 && v > SCHEMA_VERSION
231 {
232 return Err(StoreError::NewerSchema {
233 stored: v,
234 supported: SCHEMA_VERSION,
235 });
236 }
237 if stored != Some(SCHEMA_VERSION) {
238 drop(txn);
239 drop(db);
240 std::fs::remove_file(path).map_err(StoreError::Reset)?;
241 }
242 }
243 let store = Self {
244 db: Db::Writable(open_retrying(path, |p| Database::create(p))?),
245 };
246 let txn = store.db.begin_write()?;
247 {
248 let mut meta = txn.open_table(META)?;
249 meta.insert("schema", SCHEMA_VERSION)?;
250 drop(meta);
251 txn.open_table(NODES)?;
252 txn.open_table(FILE_FACTS)?;
253 txn.open_table(FILE_HASH)?;
254 txn.open_table(FILE_SCOPE)?;
255 txn.open_table(NODE_SCOPE)?;
256 txn.open_multimap_table(OUT_EDGES)?;
257 txn.open_multimap_table(IN_EDGES)?;
258 txn.open_multimap_table(UNRESOLVED)?;
259 txn.open_multimap_table(NAME_REFS)?;
260 txn.open_multimap_table(NAME_NODES)?;
261 txn.open_multimap_table(TRIGRAMS)?;
262 txn.open_multimap_table(TOKENS_WORDS)?;
263 txn.open_multimap_table(BODY_TERMS)?;
264 txn.open_multimap_table(IMPORTS)?;
265 txn.open_table(INTERN)?;
266 txn.open_table(INTERN_REV)?;
267 txn.open_table(RESOLVE_META)?;
268 txn.open_table(PENDING)?;
269 }
270 txn.commit()?;
271 Ok(store)
272 }
273
274 pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
275 Ok(Self {
276 db: Db::Writable(open_retrying(path.as_ref(), |p| Database::open(p))?),
277 })
278 }
279
280 pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self, StoreError> {
285 Ok(Self {
286 db: Db::ReadOnly(open_retrying(path.as_ref(), |p| {
287 redb::ReadOnlyDatabase::open(p)
288 })?),
289 })
290 }
291
292 pub fn is_read_only(&self) -> bool {
295 matches!(self.db, Db::ReadOnly(_))
296 }
297
298 pub fn schema(&self) -> Result<Option<u32>, StoreError> {
300 let txn = self.db.begin_read()?;
301 match txn.open_table(META) {
302 Ok(table) => Ok(table.get("schema")?.map(|g| g.value())),
303 Err(redb::TableError::TableDoesNotExist(_)) => Ok(None),
304 Err(e) => Err(e.into()),
305 }
306 }
307
308 pub fn write_graph(&self, graph: &Graph) -> Result<(), StoreError> {
311 let txn = self.db.begin_write()?;
312 {
313 let mut nodes = txn.open_table(NODES)?;
314 let mut scopes = txn.open_table(FILE_SCOPE)?;
315 let mut out = txn.open_multimap_table(OUT_EDGES)?;
316 let mut inn = txn.open_multimap_table(IN_EDGES)?;
317 for node in graph.nodes() {
318 nodes.insert(node.id.as_str(), postcard::to_allocvec(node)?.as_slice())?;
319 scopes.insert(
320 node.file.as_str(),
321 CorpusScope::classify_path(&node.file).as_str(),
322 )?;
323 }
324 for edge in graph.edges() {
325 let bytes = postcard::to_allocvec(edge)?;
326 out.insert(edge.src.as_str(), bytes.as_slice())?;
327 inn.insert(edge.dst.as_str(), bytes.as_slice())?;
328 }
329 }
330 txn.commit()?;
331 Ok(())
332 }
333
334 pub fn unresolved_count(&self) -> Result<u64, StoreError> {
336 let txn = self.db.begin_read()?;
337 let table = match txn.open_multimap_table(UNRESOLVED) {
338 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(0),
339 other => other?,
340 };
341 Ok(table.len()?)
342 }
343
344 pub fn all_unresolved(&self) -> Result<Vec<Reference>, StoreError> {
347 Ok(self
348 .all_unresolved_details()?
349 .into_iter()
350 .map(|u| u.reference)
351 .collect())
352 }
353
354 pub fn all_unresolved_details(&self) -> Result<Vec<UnresolvedReference>, StoreError> {
358 let txn = self.db.begin_read()?;
359 let table = match txn.open_multimap_table(UNRESOLVED) {
360 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
361 other => other?,
362 };
363 let mut refs = Vec::new();
364 for entry in table.iter()? {
365 let (_, values) = entry?;
366 for guard in values {
367 refs.push(postcard::from_bytes(guard?.value())?);
368 }
369 }
370 Ok(refs)
371 }
372
373 pub fn unresolved_refs(
377 &self,
378 file: Option<&str>,
379 name: Option<&str>,
380 ) -> Result<Vec<Reference>, StoreError> {
381 let mut refs = match file {
382 Some(file) => self.references_in(file)?,
383 None => self.all_unresolved()?,
384 };
385 if let Some(name) = name {
386 refs.retain(|r| name_tail_matches(&r.name, name));
387 }
388 Ok(refs)
389 }
390
391 pub fn unresolved_details(
392 &self,
393 file: Option<&str>,
394 name: Option<&str>,
395 ) -> Result<Vec<UnresolvedReference>, StoreError> {
396 let mut refs = match file {
397 Some(file) => self.unresolved_details_in(file)?,
398 None => self.all_unresolved_details()?,
399 };
400 if let Some(name) = name {
401 refs.retain(|u| name_tail_matches(&u.reference.name, name));
402 }
403 Ok(refs)
404 }
405
406 pub fn references_in(&self, file: &str) -> Result<Vec<Reference>, StoreError> {
408 Ok(self
409 .unresolved_details_in(file)?
410 .into_iter()
411 .map(|u| u.reference)
412 .collect())
413 }
414
415 pub fn unresolved_details_in(
416 &self,
417 file: &str,
418 ) -> Result<Vec<UnresolvedReference>, StoreError> {
419 let txn = self.db.begin_read()?;
420 let table = match txn.open_multimap_table(UNRESOLVED) {
421 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
422 other => other?,
423 };
424 let mut refs = Vec::new();
425 for guard in table.get(file)? {
426 refs.push(postcard::from_bytes(guard?.value())?);
427 }
428 Ok(refs)
429 }
430
431 pub fn resolve_fingerprint(&self, key: &str) -> Result<Option<String>, StoreError> {
434 let txn = self.db.begin_read()?;
435 let table = match txn.open_table(RESOLVE_META) {
436 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
437 other => other?,
438 };
439 Ok(table.get(key)?.map(|g| g.value().to_string()))
440 }
441
442 pub fn set_resolve_fingerprint(
446 &self,
447 key: &str,
448 fingerprint: Option<&str>,
449 ) -> Result<(), StoreError> {
450 if self.resolve_fingerprint(key)?.as_deref() == fingerprint {
451 return Ok(());
452 }
453 let txn = self.db.begin_write()?;
454 {
455 let mut table = txn.open_table(RESOLVE_META)?;
456 match fingerprint {
457 Some(f) => {
458 table.insert(key, f)?;
459 }
460 None => {
461 table.remove(key)?;
462 }
463 }
464 }
465 txn.commit()?;
466 Ok(())
467 }
468
469 pub fn unresolved_named(&self, name: &str) -> Result<usize, StoreError> {
473 let files = self.ref_files(&std::collections::BTreeSet::from([name.to_string()]))?;
474 let mut count = 0;
475 for file in files {
476 count += self
477 .references_in(&file)?
478 .iter()
479 .filter(|r| name_tail_matches(&r.name, name))
480 .count();
481 }
482 Ok(count)
483 }
484
485 pub fn node(&self, id: &NodeId) -> Result<Option<Node>, StoreError> {
486 let txn = self.db.begin_read()?;
487 let table = txn.open_table(NODES)?;
488 match table.get(id.as_str())? {
489 Some(guard) => Ok(Some(postcard::from_bytes(guard.value())?)),
490 None => Ok(None),
491 }
492 }
493
494 pub fn node_count(&self) -> Result<u64, StoreError> {
495 let txn = self.db.begin_read()?;
496 Ok(txn.open_table(NODES)?.len()?)
497 }
498
499 pub fn edge_count(&self) -> Result<u64, StoreError> {
500 let txn = self.db.begin_read()?;
501 Ok(txn.open_multimap_table(OUT_EDGES)?.len()?)
502 }
503
504 pub fn out_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
506 self.adjacent(OUT_EDGES, id)
507 }
508
509 pub fn in_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
511 self.adjacent(IN_EDGES, id)
512 }
513
514 pub fn in_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
517 self.adjacent_many(IN_EDGES, ids)
518 }
519
520 pub fn out_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
522 self.adjacent_many(OUT_EDGES, ids)
523 }
524
525 fn adjacent_many(
526 &self,
527 table: MultimapTableDefinition<&str, &[u8]>,
528 ids: &[NodeId],
529 ) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
530 let txn = self.db.begin_read()?;
531 let table = txn.open_multimap_table(table)?;
532 let mut found = HashMap::with_capacity(ids.len());
533 for id in ids {
534 let mut edges = Vec::new();
535 for guard in table.get(id.as_str())? {
536 edges.push(postcard::from_bytes(guard?.value())?);
537 }
538 found.insert(id.clone(), edges);
539 }
540 Ok(found)
541 }
542
543 fn adjacent(
544 &self,
545 table: MultimapTableDefinition<&str, &[u8]>,
546 id: &NodeId,
547 ) -> Result<Vec<Edge>, StoreError> {
548 let txn = self.db.begin_read()?;
549 let table = txn.open_multimap_table(table)?;
550 let mut edges = Vec::new();
551 for guard in table.get(id.as_str())? {
552 edges.push(postcard::from_bytes(guard?.value())?);
553 }
554 Ok(edges)
555 }
556
557 pub fn file_hashes(&self) -> Result<Vec<(String, FileStamp)>, StoreError> {
559 let txn = self.db.begin_read()?;
560 let table = txn.open_table(FILE_HASH)?;
561 let mut out = Vec::new();
562 for entry in table.iter()? {
563 let (k, v) = entry?;
564 out.push((k.value().to_string(), FileStamp::decode(v.value())));
565 }
566 Ok(out)
567 }
568
569 pub fn set_file_scopes(&self, rows: &[(String, CorpusScope)]) -> Result<usize, StoreError> {
575 if rows.is_empty() {
576 return Ok(0);
577 }
578 let existing = self.file_scopes()?;
579 let stale: Vec<&(String, CorpusScope)> = rows
580 .iter()
581 .filter(|(file, scope)| existing.get(file) != Some(scope))
582 .collect();
583 if stale.is_empty() {
584 return Ok(0);
585 }
586 let txn = self.db.begin_write()?;
587 {
588 let mut table = txn.open_table(FILE_SCOPE)?;
589 for (file, scope) in &stale {
590 table.insert(file.as_str(), scope.as_str())?;
591 }
592 }
593 txn.commit()?;
594 Ok(stale.len())
595 }
596
597 pub fn file_scopes(&self) -> Result<HashMap<String, CorpusScope>, StoreError> {
600 let txn = self.db.begin_read()?;
601 let table = txn.open_table(FILE_SCOPE)?;
602 let mut scopes = HashMap::new();
603 for entry in table.iter()? {
604 let (file, scope) = entry?;
605 let file = file.value().to_string();
606 scopes.insert(
607 file.clone(),
608 CorpusScope::from_str_opt(scope.value())
609 .unwrap_or_else(|| CorpusScope::classify_path(&file)),
610 );
611 }
612 Ok(scopes)
613 }
614
615 pub fn file_scope(&self, file: &str) -> Result<CorpusScope, StoreError> {
616 let txn = self.db.begin_read()?;
617 let table = txn.open_table(FILE_SCOPE)?;
618 Ok(table
619 .get(file)?
620 .and_then(|guard| CorpusScope::from_str_opt(guard.value()))
621 .unwrap_or_else(|| CorpusScope::classify_path(file)))
622 }
623
624 pub fn facts(&self, file: &str) -> Result<Option<FileFacts>, StoreError> {
625 let txn = self.db.begin_read()?;
626 let table = txn.open_table(FILE_FACTS)?;
627 match table.get(file)? {
628 Some(guard) => Ok(Some(crate::update::decode_facts(guard.value())?)),
629 None => Ok(None),
630 }
631 }
632
633 pub fn syntax_error_files(&self) -> Result<Vec<String>, StoreError> {
637 let txn = self.db.begin_read()?;
638 let table = txn.open_table(FILE_FACTS)?;
639 let mut files = Vec::new();
640 for entry in table.iter()? {
641 let (file, bytes) = entry?;
642 let facts = crate::update::decode_facts(bytes.value())?;
643 if facts.has_syntax_errors {
644 files.push(file.value().to_string());
645 }
646 }
647 files.sort();
648 Ok(files)
649 }
650
651 pub fn compact(&mut self) -> Result<bool, StoreError> {
656 let mut any = false;
657 for _ in 0..16 {
658 if !self.db.compact()? {
659 break;
660 }
661 any = true;
662 }
663 Ok(any)
664 }
665
666 pub fn all_imports(&self) -> Result<Vec<Reference>, StoreError> {
668 let txn = self.db.begin_read()?;
669 let table = txn.open_multimap_table(IMPORTS)?;
670 let mut refs = Vec::new();
671 for entry in table.iter()? {
672 let (_, values) = entry?;
673 for guard in values {
674 refs.push(postcard::from_bytes(guard?.value())?);
675 }
676 }
677 Ok(refs)
678 }
679
680 pub fn all_nodes(&self) -> Result<Vec<Node>, StoreError> {
683 let txn = self.db.begin_read()?;
684 let table = txn.open_table(NODES)?;
685 let mut nodes = Vec::new();
686 for entry in table.iter()? {
687 nodes.push(postcard::from_bytes(entry?.1.value())?);
688 }
689 Ok(nodes)
690 }
691
692 pub fn in_degrees(&self) -> Result<Vec<(String, usize)>, StoreError> {
697 let txn = self.db.begin_read()?;
698 let table = txn.open_multimap_table(IN_EDGES)?;
699 let mut out = Vec::new();
700 for entry in table.iter()? {
701 let (key, values) = entry?;
702 let mut n = 0usize;
703 for guard in values {
704 let edge: Edge = postcard::from_bytes(guard?.value())?;
705 if edge.relation != sinter_core::Relation::Contains {
706 n += 1;
707 }
708 }
709 if n > 0 {
710 out.push((key.value().to_string(), n));
711 }
712 }
713 Ok(out)
714 }
715
716 pub fn read_graph(&self) -> Result<Graph, StoreError> {
719 let txn = self.db.begin_read()?;
720 let mut graph = Graph::new();
721 {
722 let nodes = txn.open_table(NODES)?;
723 for entry in nodes.iter()? {
724 let (_, value) = entry?;
725 graph.add_node(postcard::from_bytes(value.value())?)?;
726 }
727 }
728 {
729 let out = txn.open_multimap_table(OUT_EDGES)?;
730 for entry in out.iter()? {
731 let (_, values) = entry?;
732 for guard in values {
733 graph.add_edge(postcard::from_bytes(guard?.value())?)?;
734 }
735 }
736 }
737 Ok(graph)
738 }
739}
740
741fn name_tail_matches(written: &str, name: &str) -> bool {
744 let tail = written.rsplit("::").next().unwrap_or(written);
748 let tail = tail.rsplit(['/', '.']).next().unwrap_or(tail);
749 tail == name
750}