1use std::collections::HashMap;
2use std::path::Path;
3
4use redb::{
5 Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable,
6 ReadableTableMetadata, TableDefinition,
7};
8use sinter_core::{
9 CorpusScope, Edge, FileFacts, Graph, Node, NodeId, Reference, UnresolvedReference,
10};
11
12use crate::error::StoreError;
13
14pub(crate) const NODES: TableDefinition<&str, &[u8]> = TableDefinition::new("nodes");
15pub(crate) const OUT_EDGES: MultimapTableDefinition<&str, &[u8]> =
19 MultimapTableDefinition::new("out_edges");
20pub(crate) const IN_EDGES: MultimapTableDefinition<&str, &[u8]> =
22 MultimapTableDefinition::new("in_edges");
23pub(crate) const UNRESOLVED: MultimapTableDefinition<&str, &[u8]> =
26 MultimapTableDefinition::new("unresolved");
27pub(crate) const FILE_FACTS: TableDefinition<&str, &[u8]> = TableDefinition::new("file_facts");
30pub(crate) const FILE_HASH: TableDefinition<&str, &str> = TableDefinition::new("file_hash");
32pub(crate) const FILE_SCOPE: TableDefinition<&str, &str> = TableDefinition::new("file_scope");
35pub(crate) const NODE_SCOPE: TableDefinition<&str, &str> = TableDefinition::new("node_scope");
38pub(crate) const NAME_REFS: MultimapTableDefinition<&str, &str> =
41 MultimapTableDefinition::new("name_refs");
42pub(crate) const NAME_NODES: MultimapTableDefinition<&str, u32> =
44 MultimapTableDefinition::new("name_nodes");
45pub(crate) const TRIGRAMS: MultimapTableDefinition<&str, u32> =
47 MultimapTableDefinition::new("trigrams");
48pub(crate) const TOKENS_WORDS: MultimapTableDefinition<&str, u32> =
53 MultimapTableDefinition::new("tokens_words");
54pub(crate) const BODY_TERMS: MultimapTableDefinition<&str, u32> =
58 MultimapTableDefinition::new("body_terms");
59pub(crate) const INTERN: TableDefinition<u32, &str> = TableDefinition::new("intern");
62pub(crate) const INTERN_REV: TableDefinition<&str, u32> = TableDefinition::new("intern_rev");
63pub(crate) const IMPORTS: MultimapTableDefinition<&str, &[u8]> =
66 MultimapTableDefinition::new("imports");
67pub(crate) const META: TableDefinition<&str, u32> = TableDefinition::new("meta");
70pub(crate) const RESOLVE_META: TableDefinition<&str, &str> = TableDefinition::new("resolve_meta");
75pub(crate) const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending_delta");
80const SCHEMA_VERSION: u32 = 12;
85
86#[derive(Debug, Clone, PartialEq, Eq)]
92pub struct FileStamp {
93 pub hash: String,
94 pub identity_nanos: u128,
95 pub len: u64,
96}
97
98impl FileStamp {
99 pub(crate) fn encode(&self) -> String {
100 format!("{}|{}|{}", self.hash, self.identity_nanos, self.len)
101 }
102
103 pub(crate) fn decode(value: &str) -> Self {
104 let mut parts = value.split('|');
105 let hash = parts.next().unwrap_or_default().to_string();
106 Self {
107 hash,
108 identity_nanos: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
109 len: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
111 }
112 }
113}
114
115impl Store {
116 pub const CURRENT_SCHEMA: u32 = SCHEMA_VERSION;
118
119 pub fn schema_of(path: impl AsRef<Path>) -> Result<Option<u32>, StoreError> {
122 let path = path.as_ref();
123 match Self::open_read_only(path) {
124 Ok(store) => store.schema(),
125 Err(_) => Self::open(path)?.schema(),
127 }
128 }
129}
130
131pub struct Store {
133 pub(crate) db: Db,
134}
135
136pub(crate) enum Db {
142 Writable(Database),
143 ReadOnly(redb::ReadOnlyDatabase),
144}
145
146impl Db {
147 pub(crate) fn begin_read(&self) -> Result<redb::ReadTransaction, redb::TransactionError> {
148 match self {
149 Self::Writable(db) => db.begin_read(),
150 Self::ReadOnly(db) => db.begin_read(),
151 }
152 }
153
154 pub(crate) fn begin_write(&self) -> Result<redb::WriteTransaction, StoreError> {
155 match self {
156 Self::Writable(db) => Ok(db.begin_write()?),
157 Self::ReadOnly(_) => Err(StoreError::ReadOnly),
158 }
159 }
160
161 fn compact(&mut self) -> Result<bool, StoreError> {
162 match self {
163 Self::Writable(db) => Ok(db.compact()?),
164 Self::ReadOnly(_) => Err(StoreError::ReadOnly),
165 }
166 }
167}
168
169pub(crate) fn open_retrying<D>(
177 path: &Path,
178 open: fn(&Path) -> Result<D, redb::DatabaseError>,
179) -> Result<D, redb::DatabaseError> {
180 let budget = std::time::Duration::from_secs(if cfg!(windows) { 20 } else { 5 });
181 let started = std::time::Instant::now();
182 let mut delay = std::time::Duration::from_millis(10);
183 loop {
184 match open(path) {
185 Err(redb::DatabaseError::DatabaseAlreadyOpen) if started.elapsed() < budget => {
186 std::thread::sleep(delay);
187 delay = (delay * 2).min(std::time::Duration::from_millis(200));
188 }
189 other => return other,
190 }
191 }
192}
193
194pub fn create_database(path: &Path) -> Result<Database, StoreError> {
198 Ok(open_retrying(path, |p| Database::create(p))?)
199}
200
201impl Store {
202 pub fn create(path: impl AsRef<Path>) -> Result<Self, StoreError> {
206 let path = path.as_ref();
207 if path.exists() {
208 let db = open_retrying(path, |p| Database::open(p))?;
209 let txn = db.begin_read()?;
210 let stored = match txn.open_table(META) {
211 Ok(table) => table.get("schema")?.map(|g| g.value()),
212 Err(redb::TableError::TableDoesNotExist(_)) => None,
213 Err(e) => return Err(e.into()),
214 };
215 if let Some(v) = stored
219 && v > SCHEMA_VERSION
220 {
221 return Err(StoreError::NewerSchema {
222 stored: v,
223 supported: SCHEMA_VERSION,
224 });
225 }
226 if stored != Some(SCHEMA_VERSION) {
227 drop(txn);
228 drop(db);
229 std::fs::remove_file(path).map_err(StoreError::Reset)?;
230 }
231 }
232 let store = Self {
233 db: Db::Writable(open_retrying(path, |p| Database::create(p))?),
234 };
235 let txn = store.db.begin_write()?;
236 {
237 let mut meta = txn.open_table(META)?;
238 meta.insert("schema", SCHEMA_VERSION)?;
239 drop(meta);
240 txn.open_table(NODES)?;
241 txn.open_table(FILE_FACTS)?;
242 txn.open_table(FILE_HASH)?;
243 txn.open_table(FILE_SCOPE)?;
244 txn.open_table(NODE_SCOPE)?;
245 txn.open_multimap_table(OUT_EDGES)?;
246 txn.open_multimap_table(IN_EDGES)?;
247 txn.open_multimap_table(UNRESOLVED)?;
248 txn.open_multimap_table(NAME_REFS)?;
249 txn.open_multimap_table(NAME_NODES)?;
250 txn.open_multimap_table(TRIGRAMS)?;
251 txn.open_multimap_table(TOKENS_WORDS)?;
252 txn.open_multimap_table(BODY_TERMS)?;
253 txn.open_multimap_table(IMPORTS)?;
254 txn.open_table(INTERN)?;
255 txn.open_table(INTERN_REV)?;
256 txn.open_table(RESOLVE_META)?;
257 txn.open_table(PENDING)?;
258 }
259 txn.commit()?;
260 Ok(store)
261 }
262
263 pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
264 Ok(Self {
265 db: Db::Writable(open_retrying(path.as_ref(), |p| Database::open(p))?),
266 })
267 }
268
269 pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self, StoreError> {
274 Ok(Self {
275 db: Db::ReadOnly(open_retrying(path.as_ref(), |p| {
276 redb::ReadOnlyDatabase::open(p)
277 })?),
278 })
279 }
280
281 pub fn is_read_only(&self) -> bool {
284 matches!(self.db, Db::ReadOnly(_))
285 }
286
287 pub fn schema(&self) -> Result<Option<u32>, StoreError> {
289 let txn = self.db.begin_read()?;
290 match txn.open_table(META) {
291 Ok(table) => Ok(table.get("schema")?.map(|g| g.value())),
292 Err(redb::TableError::TableDoesNotExist(_)) => Ok(None),
293 Err(e) => Err(e.into()),
294 }
295 }
296
297 pub fn write_graph(&self, graph: &Graph) -> Result<(), StoreError> {
300 let txn = self.db.begin_write()?;
301 {
302 let mut nodes = txn.open_table(NODES)?;
303 let mut scopes = txn.open_table(FILE_SCOPE)?;
304 let mut out = txn.open_multimap_table(OUT_EDGES)?;
305 let mut inn = txn.open_multimap_table(IN_EDGES)?;
306 for node in graph.nodes() {
307 nodes.insert(node.id.as_str(), postcard::to_allocvec(node)?.as_slice())?;
308 scopes.insert(
309 node.file.as_str(),
310 CorpusScope::classify_path(&node.file).as_str(),
311 )?;
312 }
313 for edge in graph.edges() {
314 let bytes = postcard::to_allocvec(edge)?;
315 out.insert(edge.src.as_str(), bytes.as_slice())?;
316 inn.insert(edge.dst.as_str(), bytes.as_slice())?;
317 }
318 }
319 txn.commit()?;
320 Ok(())
321 }
322
323 pub fn unresolved_count(&self) -> Result<u64, StoreError> {
325 let txn = self.db.begin_read()?;
326 let table = match txn.open_multimap_table(UNRESOLVED) {
327 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(0),
328 other => other?,
329 };
330 Ok(table.len()?)
331 }
332
333 pub fn all_unresolved(&self) -> Result<Vec<Reference>, StoreError> {
336 Ok(self
337 .all_unresolved_details()?
338 .into_iter()
339 .map(|u| u.reference)
340 .collect())
341 }
342
343 pub fn all_unresolved_details(&self) -> Result<Vec<UnresolvedReference>, StoreError> {
347 let txn = self.db.begin_read()?;
348 let table = match txn.open_multimap_table(UNRESOLVED) {
349 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
350 other => other?,
351 };
352 let mut refs = Vec::new();
353 for entry in table.iter()? {
354 let (_, values) = entry?;
355 for guard in values {
356 refs.push(postcard::from_bytes(guard?.value())?);
357 }
358 }
359 Ok(refs)
360 }
361
362 pub fn unresolved_refs(
366 &self,
367 file: Option<&str>,
368 name: Option<&str>,
369 ) -> Result<Vec<Reference>, StoreError> {
370 let mut refs = match file {
371 Some(file) => self.references_in(file)?,
372 None => self.all_unresolved()?,
373 };
374 if let Some(name) = name {
375 refs.retain(|r| name_tail_matches(&r.name, name));
376 }
377 Ok(refs)
378 }
379
380 pub fn unresolved_details(
381 &self,
382 file: Option<&str>,
383 name: Option<&str>,
384 ) -> Result<Vec<UnresolvedReference>, StoreError> {
385 let mut refs = match file {
386 Some(file) => self.unresolved_details_in(file)?,
387 None => self.all_unresolved_details()?,
388 };
389 if let Some(name) = name {
390 refs.retain(|u| name_tail_matches(&u.reference.name, name));
391 }
392 Ok(refs)
393 }
394
395 pub fn references_in(&self, file: &str) -> Result<Vec<Reference>, StoreError> {
397 Ok(self
398 .unresolved_details_in(file)?
399 .into_iter()
400 .map(|u| u.reference)
401 .collect())
402 }
403
404 pub fn unresolved_details_in(
405 &self,
406 file: &str,
407 ) -> Result<Vec<UnresolvedReference>, StoreError> {
408 let txn = self.db.begin_read()?;
409 let table = match txn.open_multimap_table(UNRESOLVED) {
410 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
411 other => other?,
412 };
413 let mut refs = Vec::new();
414 for guard in table.get(file)? {
415 refs.push(postcard::from_bytes(guard?.value())?);
416 }
417 Ok(refs)
418 }
419
420 pub fn resolve_fingerprint(&self, key: &str) -> Result<Option<String>, StoreError> {
423 let txn = self.db.begin_read()?;
424 let table = match txn.open_table(RESOLVE_META) {
425 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
426 other => other?,
427 };
428 Ok(table.get(key)?.map(|g| g.value().to_string()))
429 }
430
431 pub fn set_resolve_fingerprint(
435 &self,
436 key: &str,
437 fingerprint: Option<&str>,
438 ) -> Result<(), StoreError> {
439 if self.resolve_fingerprint(key)?.as_deref() == fingerprint {
440 return Ok(());
441 }
442 let txn = self.db.begin_write()?;
443 {
444 let mut table = txn.open_table(RESOLVE_META)?;
445 match fingerprint {
446 Some(f) => {
447 table.insert(key, f)?;
448 }
449 None => {
450 table.remove(key)?;
451 }
452 }
453 }
454 txn.commit()?;
455 Ok(())
456 }
457
458 pub fn unresolved_named(&self, name: &str) -> Result<usize, StoreError> {
462 let files = self.ref_files(&std::collections::BTreeSet::from([name.to_string()]))?;
463 let mut count = 0;
464 for file in files {
465 count += self
466 .references_in(&file)?
467 .iter()
468 .filter(|r| name_tail_matches(&r.name, name))
469 .count();
470 }
471 Ok(count)
472 }
473
474 pub fn node(&self, id: &NodeId) -> Result<Option<Node>, StoreError> {
475 let txn = self.db.begin_read()?;
476 let table = txn.open_table(NODES)?;
477 match table.get(id.as_str())? {
478 Some(guard) => Ok(Some(postcard::from_bytes(guard.value())?)),
479 None => Ok(None),
480 }
481 }
482
483 pub fn node_count(&self) -> Result<u64, StoreError> {
484 let txn = self.db.begin_read()?;
485 Ok(txn.open_table(NODES)?.len()?)
486 }
487
488 pub fn edge_count(&self) -> Result<u64, StoreError> {
489 let txn = self.db.begin_read()?;
490 Ok(txn.open_multimap_table(OUT_EDGES)?.len()?)
491 }
492
493 pub fn out_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
495 self.adjacent(OUT_EDGES, id)
496 }
497
498 pub fn in_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
500 self.adjacent(IN_EDGES, id)
501 }
502
503 pub fn in_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
506 self.adjacent_many(IN_EDGES, ids)
507 }
508
509 pub fn out_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
511 self.adjacent_many(OUT_EDGES, ids)
512 }
513
514 fn adjacent_many(
515 &self,
516 table: MultimapTableDefinition<&str, &[u8]>,
517 ids: &[NodeId],
518 ) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
519 let txn = self.db.begin_read()?;
520 let table = txn.open_multimap_table(table)?;
521 let mut found = HashMap::with_capacity(ids.len());
522 for id in ids {
523 let mut edges = Vec::new();
524 for guard in table.get(id.as_str())? {
525 edges.push(postcard::from_bytes(guard?.value())?);
526 }
527 found.insert(id.clone(), edges);
528 }
529 Ok(found)
530 }
531
532 fn adjacent(
533 &self,
534 table: MultimapTableDefinition<&str, &[u8]>,
535 id: &NodeId,
536 ) -> Result<Vec<Edge>, StoreError> {
537 let txn = self.db.begin_read()?;
538 let table = txn.open_multimap_table(table)?;
539 let mut edges = Vec::new();
540 for guard in table.get(id.as_str())? {
541 edges.push(postcard::from_bytes(guard?.value())?);
542 }
543 Ok(edges)
544 }
545
546 pub fn file_hashes(&self) -> Result<Vec<(String, FileStamp)>, StoreError> {
548 let txn = self.db.begin_read()?;
549 let table = txn.open_table(FILE_HASH)?;
550 let mut out = Vec::new();
551 for entry in table.iter()? {
552 let (k, v) = entry?;
553 out.push((k.value().to_string(), FileStamp::decode(v.value())));
554 }
555 Ok(out)
556 }
557
558 pub fn set_file_scopes(&self, rows: &[(String, CorpusScope)]) -> Result<usize, StoreError> {
564 if rows.is_empty() {
565 return Ok(0);
566 }
567 let existing = self.file_scopes()?;
568 let stale: Vec<&(String, CorpusScope)> = rows
569 .iter()
570 .filter(|(file, scope)| existing.get(file) != Some(scope))
571 .collect();
572 if stale.is_empty() {
573 return Ok(0);
574 }
575 let txn = self.db.begin_write()?;
576 {
577 let mut table = txn.open_table(FILE_SCOPE)?;
578 for (file, scope) in &stale {
579 table.insert(file.as_str(), scope.as_str())?;
580 }
581 }
582 txn.commit()?;
583 Ok(stale.len())
584 }
585
586 pub fn file_scopes(&self) -> Result<HashMap<String, CorpusScope>, StoreError> {
589 let txn = self.db.begin_read()?;
590 let table = txn.open_table(FILE_SCOPE)?;
591 let mut scopes = HashMap::new();
592 for entry in table.iter()? {
593 let (file, scope) = entry?;
594 let file = file.value().to_string();
595 scopes.insert(
596 file.clone(),
597 CorpusScope::from_str_opt(scope.value())
598 .unwrap_or_else(|| CorpusScope::classify_path(&file)),
599 );
600 }
601 Ok(scopes)
602 }
603
604 pub fn file_scope(&self, file: &str) -> Result<CorpusScope, StoreError> {
605 let txn = self.db.begin_read()?;
606 let table = txn.open_table(FILE_SCOPE)?;
607 Ok(table
608 .get(file)?
609 .and_then(|guard| CorpusScope::from_str_opt(guard.value()))
610 .unwrap_or_else(|| CorpusScope::classify_path(file)))
611 }
612
613 pub fn facts(&self, file: &str) -> Result<Option<FileFacts>, StoreError> {
614 let txn = self.db.begin_read()?;
615 let table = txn.open_table(FILE_FACTS)?;
616 match table.get(file)? {
617 Some(guard) => Ok(Some(crate::update::decode_facts(guard.value())?)),
618 None => Ok(None),
619 }
620 }
621
622 pub fn syntax_error_files(&self) -> Result<Vec<String>, StoreError> {
626 let txn = self.db.begin_read()?;
627 let table = txn.open_table(FILE_FACTS)?;
628 let mut files = Vec::new();
629 for entry in table.iter()? {
630 let (file, bytes) = entry?;
631 let facts = crate::update::decode_facts(bytes.value())?;
632 if facts.has_syntax_errors {
633 files.push(file.value().to_string());
634 }
635 }
636 files.sort();
637 Ok(files)
638 }
639
640 pub fn compact(&mut self) -> Result<bool, StoreError> {
645 let mut any = false;
646 for _ in 0..16 {
647 if !self.db.compact()? {
648 break;
649 }
650 any = true;
651 }
652 Ok(any)
653 }
654
655 pub fn all_imports(&self) -> Result<Vec<Reference>, StoreError> {
657 let txn = self.db.begin_read()?;
658 let table = txn.open_multimap_table(IMPORTS)?;
659 let mut refs = Vec::new();
660 for entry in table.iter()? {
661 let (_, values) = entry?;
662 for guard in values {
663 refs.push(postcard::from_bytes(guard?.value())?);
664 }
665 }
666 Ok(refs)
667 }
668
669 pub fn all_nodes(&self) -> Result<Vec<Node>, StoreError> {
672 let txn = self.db.begin_read()?;
673 let table = txn.open_table(NODES)?;
674 let mut nodes = Vec::new();
675 for entry in table.iter()? {
676 nodes.push(postcard::from_bytes(entry?.1.value())?);
677 }
678 Ok(nodes)
679 }
680
681 pub fn in_degrees(&self) -> Result<Vec<(String, usize)>, StoreError> {
686 let txn = self.db.begin_read()?;
687 let table = txn.open_multimap_table(IN_EDGES)?;
688 let mut out = Vec::new();
689 for entry in table.iter()? {
690 let (key, values) = entry?;
691 let mut n = 0usize;
692 for guard in values {
693 let edge: Edge = postcard::from_bytes(guard?.value())?;
694 if edge.relation != sinter_core::Relation::Contains {
695 n += 1;
696 }
697 }
698 if n > 0 {
699 out.push((key.value().to_string(), n));
700 }
701 }
702 Ok(out)
703 }
704
705 pub fn read_graph(&self) -> Result<Graph, StoreError> {
708 let txn = self.db.begin_read()?;
709 let mut graph = Graph::new();
710 {
711 let nodes = txn.open_table(NODES)?;
712 for entry in nodes.iter()? {
713 let (_, value) = entry?;
714 graph.add_node(postcard::from_bytes(value.value())?)?;
715 }
716 }
717 {
718 let out = txn.open_multimap_table(OUT_EDGES)?;
719 for entry in out.iter()? {
720 let (_, values) = entry?;
721 for guard in values {
722 graph.add_edge(postcard::from_bytes(guard?.value())?)?;
723 }
724 }
725 }
726 Ok(graph)
727 }
728}
729
730fn name_tail_matches(written: &str, name: &str) -> bool {
733 let tail = written.rsplit("::").next().unwrap_or(written);
737 let tail = tail.rsplit(['/', '.']).next().unwrap_or(tail);
738 tail == name
739}