1use std::collections::HashMap;
2use std::path::Path;
3
4use redb::{
5 Database, MultimapTableDefinition, ReadableDatabase, ReadableMultimapTable, ReadableTable,
6 ReadableTableMetadata, TableDefinition,
7};
8use sinter_core::{
9 CorpusScope, Edge, FileFacts, Graph, Node, NodeId, Reference, UnresolvedReference,
10};
11
12use crate::error::StoreError;
13
14pub(crate) const NODES: TableDefinition<&str, &[u8]> = TableDefinition::new("nodes");
15pub(crate) const OUT_EDGES: MultimapTableDefinition<&str, &[u8]> =
19 MultimapTableDefinition::new("out_edges");
20pub(crate) const IN_EDGES: MultimapTableDefinition<&str, &[u8]> =
22 MultimapTableDefinition::new("in_edges");
23pub(crate) const UNRESOLVED: MultimapTableDefinition<&str, &[u8]> =
26 MultimapTableDefinition::new("unresolved");
27pub(crate) const FILE_FACTS: TableDefinition<&str, &[u8]> = TableDefinition::new("file_facts");
30pub(crate) const FILE_HASH: TableDefinition<&str, &str> = TableDefinition::new("file_hash");
32pub(crate) const FILE_SCOPE: TableDefinition<&str, &str> = TableDefinition::new("file_scope");
35pub(crate) const NAME_REFS: MultimapTableDefinition<&str, &str> =
38 MultimapTableDefinition::new("name_refs");
39pub(crate) const NAME_NODES: MultimapTableDefinition<&str, u32> =
41 MultimapTableDefinition::new("name_nodes");
42pub(crate) const TRIGRAMS: MultimapTableDefinition<&str, u32> =
44 MultimapTableDefinition::new("trigrams");
45pub(crate) const TOKENS_WORDS: MultimapTableDefinition<&str, u32> =
50 MultimapTableDefinition::new("tokens_words");
51pub(crate) const INTERN: TableDefinition<u32, &str> = TableDefinition::new("intern");
54pub(crate) const INTERN_REV: TableDefinition<&str, u32> = TableDefinition::new("intern_rev");
55pub(crate) const IMPORTS: MultimapTableDefinition<&str, &[u8]> =
58 MultimapTableDefinition::new("imports");
59pub(crate) const META: TableDefinition<&str, u32> = TableDefinition::new("meta");
62pub(crate) const RESOLVE_META: TableDefinition<&str, &str> = TableDefinition::new("resolve_meta");
67pub(crate) const PENDING: TableDefinition<&str, &[u8]> = TableDefinition::new("pending_delta");
72const SCHEMA_VERSION: u32 = 10;
75
76#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct FileStamp {
83 pub hash: String,
84 pub identity_nanos: u128,
85 pub len: u64,
86}
87
88impl FileStamp {
89 pub(crate) fn encode(&self) -> String {
90 format!("{}|{}|{}", self.hash, self.identity_nanos, self.len)
91 }
92
93 pub(crate) fn decode(value: &str) -> Self {
94 let mut parts = value.split('|');
95 let hash = parts.next().unwrap_or_default().to_string();
96 Self {
97 hash,
98 identity_nanos: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
99 len: parts.next().and_then(|p| p.parse().ok()).unwrap_or(0),
101 }
102 }
103}
104
105impl Store {
106 pub const CURRENT_SCHEMA: u32 = SCHEMA_VERSION;
108
109 pub fn schema_of(path: impl AsRef<Path>) -> Result<Option<u32>, StoreError> {
112 Self::open(path)?.schema()
113 }
114}
115
116pub struct Store {
118 pub(crate) db: Database,
119}
120
121pub(crate) fn open_retrying(
129 path: &Path,
130 open: fn(&Path) -> Result<Database, redb::DatabaseError>,
131) -> Result<Database, redb::DatabaseError> {
132 let budget = std::time::Duration::from_secs(if cfg!(windows) { 20 } else { 5 });
133 let started = std::time::Instant::now();
134 let mut delay = std::time::Duration::from_millis(10);
135 loop {
136 match open(path) {
137 Err(redb::DatabaseError::DatabaseAlreadyOpen) if started.elapsed() < budget => {
138 std::thread::sleep(delay);
139 delay = (delay * 2).min(std::time::Duration::from_millis(200));
140 }
141 other => return other,
142 }
143 }
144}
145
146pub fn create_database(path: &Path) -> Result<Database, StoreError> {
150 Ok(open_retrying(path, |p| Database::create(p))?)
151}
152
153impl Store {
154 pub fn create(path: impl AsRef<Path>) -> Result<Self, StoreError> {
158 let path = path.as_ref();
159 if path.exists() {
160 let db = open_retrying(path, |p| Database::open(p))?;
161 let txn = db.begin_read()?;
162 let stored = match txn.open_table(META) {
163 Ok(table) => table.get("schema")?.map(|g| g.value()),
164 Err(redb::TableError::TableDoesNotExist(_)) => None,
165 Err(e) => return Err(e.into()),
166 };
167 if let Some(v) = stored
171 && v > SCHEMA_VERSION
172 {
173 return Err(StoreError::NewerSchema {
174 stored: v,
175 supported: SCHEMA_VERSION,
176 });
177 }
178 if stored != Some(SCHEMA_VERSION) {
179 drop(txn);
180 drop(db);
181 std::fs::remove_file(path).map_err(StoreError::Reset)?;
182 }
183 }
184 let store = Self {
185 db: open_retrying(path, |p| Database::create(p))?,
186 };
187 let txn = store.db.begin_write()?;
188 {
189 let mut meta = txn.open_table(META)?;
190 meta.insert("schema", SCHEMA_VERSION)?;
191 drop(meta);
192 txn.open_table(NODES)?;
193 txn.open_table(FILE_FACTS)?;
194 txn.open_table(FILE_HASH)?;
195 txn.open_table(FILE_SCOPE)?;
196 txn.open_multimap_table(OUT_EDGES)?;
197 txn.open_multimap_table(IN_EDGES)?;
198 txn.open_multimap_table(UNRESOLVED)?;
199 txn.open_multimap_table(NAME_REFS)?;
200 txn.open_multimap_table(NAME_NODES)?;
201 txn.open_multimap_table(TRIGRAMS)?;
202 txn.open_multimap_table(TOKENS_WORDS)?;
203 txn.open_multimap_table(IMPORTS)?;
204 txn.open_table(INTERN)?;
205 txn.open_table(INTERN_REV)?;
206 txn.open_table(RESOLVE_META)?;
207 txn.open_table(PENDING)?;
208 }
209 txn.commit()?;
210 Ok(store)
211 }
212
213 pub fn open(path: impl AsRef<Path>) -> Result<Self, StoreError> {
214 Ok(Self {
215 db: open_retrying(path.as_ref(), |p| Database::open(p))?,
216 })
217 }
218
219 pub fn schema(&self) -> Result<Option<u32>, StoreError> {
221 let txn = self.db.begin_read()?;
222 match txn.open_table(META) {
223 Ok(table) => Ok(table.get("schema")?.map(|g| g.value())),
224 Err(redb::TableError::TableDoesNotExist(_)) => Ok(None),
225 Err(e) => Err(e.into()),
226 }
227 }
228
229 pub fn write_graph(&self, graph: &Graph) -> Result<(), StoreError> {
232 let txn = self.db.begin_write()?;
233 {
234 let mut nodes = txn.open_table(NODES)?;
235 let mut scopes = txn.open_table(FILE_SCOPE)?;
236 let mut out = txn.open_multimap_table(OUT_EDGES)?;
237 let mut inn = txn.open_multimap_table(IN_EDGES)?;
238 for node in graph.nodes() {
239 nodes.insert(node.id.as_str(), postcard::to_allocvec(node)?.as_slice())?;
240 scopes.insert(
241 node.file.as_str(),
242 CorpusScope::classify_path(&node.file).as_str(),
243 )?;
244 }
245 for edge in graph.edges() {
246 let bytes = postcard::to_allocvec(edge)?;
247 out.insert(edge.src.as_str(), bytes.as_slice())?;
248 inn.insert(edge.dst.as_str(), bytes.as_slice())?;
249 }
250 }
251 txn.commit()?;
252 Ok(())
253 }
254
255 pub fn unresolved_count(&self) -> Result<u64, StoreError> {
257 let txn = self.db.begin_read()?;
258 let table = match txn.open_multimap_table(UNRESOLVED) {
259 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(0),
260 other => other?,
261 };
262 Ok(table.len()?)
263 }
264
265 pub fn all_unresolved(&self) -> Result<Vec<Reference>, StoreError> {
268 Ok(self
269 .all_unresolved_details()?
270 .into_iter()
271 .map(|u| u.reference)
272 .collect())
273 }
274
275 pub fn all_unresolved_details(&self) -> Result<Vec<UnresolvedReference>, StoreError> {
279 let txn = self.db.begin_read()?;
280 let table = match txn.open_multimap_table(UNRESOLVED) {
281 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
282 other => other?,
283 };
284 let mut refs = Vec::new();
285 for entry in table.iter()? {
286 let (_, values) = entry?;
287 for guard in values {
288 refs.push(postcard::from_bytes(guard?.value())?);
289 }
290 }
291 Ok(refs)
292 }
293
294 pub fn unresolved_refs(
298 &self,
299 file: Option<&str>,
300 name: Option<&str>,
301 ) -> Result<Vec<Reference>, StoreError> {
302 let mut refs = match file {
303 Some(file) => self.references_in(file)?,
304 None => self.all_unresolved()?,
305 };
306 if let Some(name) = name {
307 refs.retain(|r| name_tail_matches(&r.name, name));
308 }
309 Ok(refs)
310 }
311
312 pub fn unresolved_details(
313 &self,
314 file: Option<&str>,
315 name: Option<&str>,
316 ) -> Result<Vec<UnresolvedReference>, StoreError> {
317 let mut refs = match file {
318 Some(file) => self.unresolved_details_in(file)?,
319 None => self.all_unresolved_details()?,
320 };
321 if let Some(name) = name {
322 refs.retain(|u| name_tail_matches(&u.reference.name, name));
323 }
324 Ok(refs)
325 }
326
327 pub fn references_in(&self, file: &str) -> Result<Vec<Reference>, StoreError> {
329 Ok(self
330 .unresolved_details_in(file)?
331 .into_iter()
332 .map(|u| u.reference)
333 .collect())
334 }
335
336 pub fn unresolved_details_in(
337 &self,
338 file: &str,
339 ) -> Result<Vec<UnresolvedReference>, StoreError> {
340 let txn = self.db.begin_read()?;
341 let table = match txn.open_multimap_table(UNRESOLVED) {
342 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(Vec::new()),
343 other => other?,
344 };
345 let mut refs = Vec::new();
346 for guard in table.get(file)? {
347 refs.push(postcard::from_bytes(guard?.value())?);
348 }
349 Ok(refs)
350 }
351
352 pub fn resolve_fingerprint(&self, key: &str) -> Result<Option<String>, StoreError> {
355 let txn = self.db.begin_read()?;
356 let table = match txn.open_table(RESOLVE_META) {
357 Err(redb::TableError::TableDoesNotExist(_)) => return Ok(None),
358 other => other?,
359 };
360 Ok(table.get(key)?.map(|g| g.value().to_string()))
361 }
362
363 pub fn set_resolve_fingerprint(
367 &self,
368 key: &str,
369 fingerprint: Option<&str>,
370 ) -> Result<(), StoreError> {
371 if self.resolve_fingerprint(key)?.as_deref() == fingerprint {
372 return Ok(());
373 }
374 let txn = self.db.begin_write()?;
375 {
376 let mut table = txn.open_table(RESOLVE_META)?;
377 match fingerprint {
378 Some(f) => {
379 table.insert(key, f)?;
380 }
381 None => {
382 table.remove(key)?;
383 }
384 }
385 }
386 txn.commit()?;
387 Ok(())
388 }
389
390 pub fn unresolved_named(&self, name: &str) -> Result<usize, StoreError> {
394 let files = self.ref_files(&std::collections::BTreeSet::from([name.to_string()]))?;
395 let mut count = 0;
396 for file in files {
397 count += self
398 .references_in(&file)?
399 .iter()
400 .filter(|r| name_tail_matches(&r.name, name))
401 .count();
402 }
403 Ok(count)
404 }
405
406 pub fn node(&self, id: &NodeId) -> Result<Option<Node>, StoreError> {
407 let txn = self.db.begin_read()?;
408 let table = txn.open_table(NODES)?;
409 match table.get(id.as_str())? {
410 Some(guard) => Ok(Some(postcard::from_bytes(guard.value())?)),
411 None => Ok(None),
412 }
413 }
414
415 pub fn node_count(&self) -> Result<u64, StoreError> {
416 let txn = self.db.begin_read()?;
417 Ok(txn.open_table(NODES)?.len()?)
418 }
419
420 pub fn edge_count(&self) -> Result<u64, StoreError> {
421 let txn = self.db.begin_read()?;
422 Ok(txn.open_multimap_table(OUT_EDGES)?.len()?)
423 }
424
425 pub fn out_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
427 self.adjacent(OUT_EDGES, id)
428 }
429
430 pub fn in_edges(&self, id: &NodeId) -> Result<Vec<Edge>, StoreError> {
432 self.adjacent(IN_EDGES, id)
433 }
434
435 pub fn in_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
438 self.adjacent_many(IN_EDGES, ids)
439 }
440
441 pub fn out_edges_many(&self, ids: &[NodeId]) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
443 self.adjacent_many(OUT_EDGES, ids)
444 }
445
446 fn adjacent_many(
447 &self,
448 table: MultimapTableDefinition<&str, &[u8]>,
449 ids: &[NodeId],
450 ) -> Result<HashMap<NodeId, Vec<Edge>>, StoreError> {
451 let txn = self.db.begin_read()?;
452 let table = txn.open_multimap_table(table)?;
453 let mut found = HashMap::with_capacity(ids.len());
454 for id in ids {
455 let mut edges = Vec::new();
456 for guard in table.get(id.as_str())? {
457 edges.push(postcard::from_bytes(guard?.value())?);
458 }
459 found.insert(id.clone(), edges);
460 }
461 Ok(found)
462 }
463
464 fn adjacent(
465 &self,
466 table: MultimapTableDefinition<&str, &[u8]>,
467 id: &NodeId,
468 ) -> Result<Vec<Edge>, StoreError> {
469 let txn = self.db.begin_read()?;
470 let table = txn.open_multimap_table(table)?;
471 let mut edges = Vec::new();
472 for guard in table.get(id.as_str())? {
473 edges.push(postcard::from_bytes(guard?.value())?);
474 }
475 Ok(edges)
476 }
477
478 pub fn file_hashes(&self) -> Result<Vec<(String, FileStamp)>, StoreError> {
480 let txn = self.db.begin_read()?;
481 let table = txn.open_table(FILE_HASH)?;
482 let mut out = Vec::new();
483 for entry in table.iter()? {
484 let (k, v) = entry?;
485 out.push((k.value().to_string(), FileStamp::decode(v.value())));
486 }
487 Ok(out)
488 }
489
490 pub fn set_file_scopes(&self, rows: &[(String, CorpusScope)]) -> Result<(), StoreError> {
493 if rows.is_empty() {
494 return Ok(());
495 }
496 let existing = self.file_scopes()?;
497 if rows
498 .iter()
499 .all(|(file, scope)| existing.get(file) == Some(scope))
500 {
501 return Ok(());
502 }
503 let txn = self.db.begin_write()?;
504 {
505 let mut table = txn.open_table(FILE_SCOPE)?;
506 for (file, scope) in rows {
507 table.insert(file.as_str(), scope.as_str())?;
508 }
509 }
510 txn.commit()?;
511 Ok(())
512 }
513
514 pub fn file_scopes(&self) -> Result<HashMap<String, CorpusScope>, StoreError> {
517 let txn = self.db.begin_read()?;
518 let table = txn.open_table(FILE_SCOPE)?;
519 let mut scopes = HashMap::new();
520 for entry in table.iter()? {
521 let (file, scope) = entry?;
522 let file = file.value().to_string();
523 scopes.insert(
524 file.clone(),
525 CorpusScope::from_str_opt(scope.value())
526 .unwrap_or_else(|| CorpusScope::classify_path(&file)),
527 );
528 }
529 Ok(scopes)
530 }
531
532 pub fn file_scope(&self, file: &str) -> Result<CorpusScope, StoreError> {
533 let txn = self.db.begin_read()?;
534 let table = txn.open_table(FILE_SCOPE)?;
535 Ok(table
536 .get(file)?
537 .and_then(|guard| CorpusScope::from_str_opt(guard.value()))
538 .unwrap_or_else(|| CorpusScope::classify_path(file)))
539 }
540
541 pub fn facts(&self, file: &str) -> Result<Option<FileFacts>, StoreError> {
542 let txn = self.db.begin_read()?;
543 let table = txn.open_table(FILE_FACTS)?;
544 match table.get(file)? {
545 Some(guard) => Ok(Some(crate::update::decode_facts(guard.value())?)),
546 None => Ok(None),
547 }
548 }
549
550 pub fn syntax_error_files(&self) -> Result<Vec<String>, StoreError> {
554 let txn = self.db.begin_read()?;
555 let table = txn.open_table(FILE_FACTS)?;
556 let mut files = Vec::new();
557 for entry in table.iter()? {
558 let (file, bytes) = entry?;
559 let facts = crate::update::decode_facts(bytes.value())?;
560 if facts.has_syntax_errors {
561 files.push(file.value().to_string());
562 }
563 }
564 files.sort();
565 Ok(files)
566 }
567
568 pub fn compact(&mut self) -> Result<bool, StoreError> {
573 let mut any = false;
574 for _ in 0..16 {
575 if !self.db.compact()? {
576 break;
577 }
578 any = true;
579 }
580 Ok(any)
581 }
582
583 pub fn all_imports(&self) -> Result<Vec<Reference>, StoreError> {
585 let txn = self.db.begin_read()?;
586 let table = txn.open_multimap_table(IMPORTS)?;
587 let mut refs = Vec::new();
588 for entry in table.iter()? {
589 let (_, values) = entry?;
590 for guard in values {
591 refs.push(postcard::from_bytes(guard?.value())?);
592 }
593 }
594 Ok(refs)
595 }
596
597 pub fn all_nodes(&self) -> Result<Vec<Node>, StoreError> {
600 let txn = self.db.begin_read()?;
601 let table = txn.open_table(NODES)?;
602 let mut nodes = Vec::new();
603 for entry in table.iter()? {
604 nodes.push(postcard::from_bytes(entry?.1.value())?);
605 }
606 Ok(nodes)
607 }
608
609 pub fn in_degrees(&self) -> Result<Vec<(String, usize)>, StoreError> {
614 let txn = self.db.begin_read()?;
615 let table = txn.open_multimap_table(IN_EDGES)?;
616 let mut out = Vec::new();
617 for entry in table.iter()? {
618 let (key, values) = entry?;
619 let mut n = 0usize;
620 for guard in values {
621 let edge: Edge = postcard::from_bytes(guard?.value())?;
622 if edge.relation != sinter_core::Relation::Contains {
623 n += 1;
624 }
625 }
626 if n > 0 {
627 out.push((key.value().to_string(), n));
628 }
629 }
630 Ok(out)
631 }
632
633 pub fn read_graph(&self) -> Result<Graph, StoreError> {
636 let txn = self.db.begin_read()?;
637 let mut graph = Graph::new();
638 {
639 let nodes = txn.open_table(NODES)?;
640 for entry in nodes.iter()? {
641 let (_, value) = entry?;
642 graph.add_node(postcard::from_bytes(value.value())?)?;
643 }
644 }
645 {
646 let out = txn.open_multimap_table(OUT_EDGES)?;
647 for entry in out.iter()? {
648 let (_, values) = entry?;
649 for guard in values {
650 graph.add_edge(postcard::from_bytes(guard?.value())?)?;
651 }
652 }
653 }
654 Ok(graph)
655 }
656}
657
658fn name_tail_matches(written: &str, name: &str) -> bool {
661 let tail = written.rsplit("::").next().unwrap_or(written);
665 let tail = tail.rsplit(['/', '.']).next().unwrap_or(tail);
666 tail == name
667}