concept_store/
store.rs

1//! Read-only access to a built artifact.
2//!
3//! Every method is a point read: one position or one key, one record, decoded
4//! with a typed error. Ordinal-keyed data is answered from a dense column, read
5//! at open from its own file beside the database; the designation text is
6//! point-read from the database, one row per concept. Scans belong to the
7//! offline build, never to a request path.
8//!
9//! Opening takes one read snapshot and keeps the two tables a request path
10//! reads open on it, so a lookup costs one descent rather than a transaction,
11//! a table-name lookup, and a descent.
12
13use std::path::{Path, PathBuf};
14
15use concept_graph::ordinal::Ordinal;
16use redb::{ReadOnlyDatabase, ReadOnlyTable, ReadableDatabase, ReadableTable, TableHandle};
17
18use crate::column::{Column, ColumnError};
19use crate::record::{self, Concept, Designation, PropertyValue, RecordError};
20use crate::tables;
21
22/// A failure while opening or reading an artifact.
23#[derive(Debug, thiserror::Error)]
24pub enum StoreError {
25    /// The database could not be opened or read.
26    #[error("cannot open the artifact at {path}")]
27    Open {
28        /// The artifact.
29        path: PathBuf,
30        /// The underlying error.
31        #[source]
32        source: redb::DatabaseError,
33    },
34    /// A transaction failed.
35    #[error("cannot begin a read transaction")]
36    Transaction(#[from] redb::TransactionError),
37    /// A table is missing or has another type.
38    #[error("cannot open table {table}")]
39    Table {
40        /// The table name.
41        table: String,
42        /// The underlying error.
43        #[source]
44        source: redb::TableError,
45    },
46    /// A read failed inside the database.
47    #[error("storage read failed")]
48    Storage(#[from] redb::StorageError),
49    /// The artifact was written by another layout version.
50    #[error("artifact layout {found:?}, expected {expected:?}")]
51    Layout {
52        /// The layout found, if any.
53        found: Option<String>,
54        /// The layout this build reads.
55        expected: &'static str,
56    },
57    /// A column's side file is missing or is not the layout this build reads.
58    #[error("the {column} column does not read from {path}")]
59    Column {
60        /// The column name.
61        column: String,
62        /// The side file it was read from.
63        path: PathBuf,
64        /// The underlying error.
65        #[source]
66        source: ColumnError,
67    },
68    /// A stored record is damaged.
69    #[error("damaged record in table {table} at key {key}")]
70    Record {
71        /// The table name.
72        table: String,
73        /// The key, rendered.
74        key: String,
75        /// The underlying error.
76        #[source]
77        source: RecordError,
78    },
79}
80
81/// How much memory the database may keep for the pages it has read.
82///
83// NOTE: redb's default is a gibibyte
84// (<https://docs.rs/redb/4.2.0/redb/struct.Builder.html#method.set_cache_size>),
85// which a store answering designation rows at random would fill and hold.
86const CACHE_BYTES: usize = 64 << 20;
87
88/// An opened artifact.
89pub struct Store {
90    path: PathBuf,
91    db: ReadOnlyDatabase,
92    /// The concepts, read once at open. An ordinal is a position, so this
93    /// column answers by slicing where a b-tree would search.
94    concepts: Column,
95    /// The preferred designations the build chose, per concept, so choosing a
96    /// display reads no table at all.
97    displays: Column,
98    /// The properties of every concept. The whole hierarchy is read through
99    /// these on an ECL refinement, so they are answered by position.
100    properties: Column,
101    /// The acceptability of every designation, per language reference set.
102    acceptability: Column,
103    /// The native code to ordinal index, held open on the snapshot the store
104    /// was opened on. A string key is what a b-tree is for; paying for a
105    /// transaction and a table-name lookup on top of it is not.
106    code_index: ReadOnlyTable<&'static str, u32>,
107    /// The designations of every concept, one row per concept, held open on
108    /// the same snapshot. The text is the largest thing an artifact holds, so
109    /// it stays on disk rather than becoming a resident column (#338).
110    designation_rows: ReadOnlyTable<u32, &'static [u8]>,
111    /// The acceptability ordinal that marks a designation preferred.
112    preferred: Option<u32>,
113}
114
115impl std::fmt::Debug for Store {
116    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
117        f.debug_struct("Store")
118            .field("path", &self.path)
119            .finish_non_exhaustive()
120    }
121}
122
123macro_rules! open_table {
124    ($txn:expr, $def:expr) => {
125        $txn.open_table($def).map_err(|source| StoreError::Table {
126            table: $def.name().to_owned(),
127            source,
128        })
129    };
130}
131
132impl Store {
133    /// Opens the artifact at `path` read-only and checks its layout version.
134    ///
135    /// The four dense columns are read from their own files beside `path`, so
136    /// each one arrives at the address it is served from.
137    ///
138    /// # Errors
139    ///
140    /// Returns [`StoreError`] when the file cannot be opened, is not an
141    /// artifact of this layout, or a table or a column file is missing.
142    pub fn open(path: &Path) -> Result<Self, StoreError> {
143        let db = redb::Builder::new()
144            .set_cache_size(CACHE_BYTES)
145            .open_read_only(path)
146            .map_err(|source| StoreError::Open {
147                path: path.to_path_buf(),
148                source,
149            })?;
150        let txn = db.begin_read()?;
151        let meta = open_table!(txn, tables::META)?;
152        let layout = meta
153            .get(tables::META_LAYOUT)?
154            .map(|value| value.value().to_owned());
155        if layout.as_deref() != Some(tables::LAYOUT_VERSION) {
156            return Err(StoreError::Layout {
157                found: layout,
158                expected: tables::LAYOUT_VERSION,
159            });
160        }
161        let column = |name: &str| -> Result<Column, StoreError> {
162            let file = Column::file(path, name);
163            let failed = |source| StoreError::Column {
164                column: name.to_owned(),
165                path: file.clone(),
166                source,
167            };
168            let mut reader = concept_graph::read::buffered(&file)
169                .map_err(|source| failed(ColumnError::Io(source)))?;
170            Column::read_from(&mut reader).map_err(failed)
171        };
172        let concepts = column(tables::COLUMN_CONCEPTS)?;
173        let displays = column(tables::COLUMN_DISPLAYS)?;
174        let properties = column(tables::COLUMN_PROPERTIES)?;
175        let acceptability = column(tables::COLUMN_ACCEPTABILITY)?;
176        let preferred = meta
177            .get(tables::META_PREFERRED)?
178            .and_then(|value| value.value().parse().ok());
179        let code_index = open_table!(txn, tables::CODES)?;
180        let designation_rows = open_table!(txn, tables::DESIGNATIONS)?;
181        Ok(Self {
182            path: path.to_path_buf(),
183            db,
184            concepts,
185            displays,
186            properties,
187            acceptability,
188            code_index,
189            designation_rows,
190            preferred,
191        })
192    }
193
194    /// The artifact's path.
195    #[must_use]
196    pub fn path(&self) -> &Path {
197        &self.path
198    }
199
200    /// The heap bytes each resident column holds, by column name.
201    ///
202    /// The designation text and the code index stay in the database and are
203    /// point-read, so they are not here; what `redb` keeps in its page cache
204    /// is the database's own and no column of it.
205    #[must_use]
206    pub fn column_sizes(&self) -> [(&'static str, usize); 4] {
207        [
208            (tables::COLUMN_CONCEPTS, self.concepts.size_in_bytes()),
209            (tables::COLUMN_DISPLAYS, self.displays.size_in_bytes()),
210            (tables::COLUMN_PROPERTIES, self.properties.size_in_bytes()),
211            (
212                tables::COLUMN_ACCEPTABILITY,
213                self.acceptability.size_in_bytes(),
214            ),
215        ]
216    }
217
218    /// The heap bytes the resident columns hold together.
219    #[must_use]
220    pub fn size_in_bytes(&self) -> usize {
221        self.column_sizes()
222            .into_iter()
223            .fold(0_usize, |total, (_, bytes)| total.saturating_add(bytes))
224    }
225
226    /// An artifact-level fact by key.
227    ///
228    /// # Errors
229    ///
230    /// Returns [`StoreError`] when the database cannot be read.
231    pub fn meta(&self, key: &str) -> Result<Option<String>, StoreError> {
232        let txn = self.db.begin_read()?;
233        let table = open_table!(txn, tables::META)?;
234        Ok(table.get(key)?.map(|v| v.value().to_owned()))
235    }
236
237    /// The ordinal of a native code, if the version has it.
238    ///
239    /// # Errors
240    ///
241    /// Returns [`StoreError`] when the database cannot be read.
242    pub fn ordinal(&self, code: &str) -> Result<Option<Ordinal>, StoreError> {
243        Ok(self
244            .code_index
245            .get(code)?
246            .map(|value| Ordinal::new(value.value())))
247    }
248
249    /// The concept at `ordinal`, if any.
250    ///
251    /// # Errors
252    ///
253    /// Returns [`StoreError`] when the database cannot be read or the record is damaged.
254    pub fn concept(&self, ordinal: Ordinal) -> Result<Option<Concept>, StoreError> {
255        self.concepts
256            .get(ordinal)
257            .map(|bytes| Self::decode_concept(ordinal, bytes))
258            .transpose()
259    }
260
261    /// The native code at `ordinal`, borrowed from the column.
262    ///
263    /// The code is the first field of the record, so this reads one length
264    /// and one string and copies nothing.
265    ///
266    /// # Errors
267    ///
268    /// Returns [`StoreError`] when the code is damaged.
269    pub fn code(&self, ordinal: Ordinal) -> Result<Option<&str>, StoreError> {
270        self.concepts
271            .get(ordinal)
272            .map(|bytes| {
273                record::code(bytes).map_err(|source| StoreError::Record {
274                    table: tables::COLUMN_CONCEPTS.to_owned(),
275                    key: ordinal.to_string(),
276                    source,
277                })
278            })
279            .transpose()
280    }
281
282    /// Decodes one concept record, naming the ordinal it came from.
283    fn decode_concept(ordinal: Ordinal, bytes: &[u8]) -> Result<Concept, StoreError> {
284        Concept::decode(bytes).map_err(|source| StoreError::Record {
285            table: tables::COLUMN_CONCEPTS.to_owned(),
286            key: ordinal.to_string(),
287            source,
288        })
289    }
290
291    /// The concepts at `ordinals`, in the order given, `None` where the store
292    /// has no such concept.
293    ///
294    /// # Errors
295    ///
296    /// Returns [`StoreError`] when a record is damaged.
297    pub fn concepts(
298        &self,
299        ordinals: impl IntoIterator<Item = Ordinal>,
300    ) -> Result<Vec<Option<Concept>>, StoreError> {
301        ordinals
302            .into_iter()
303            .map(|ordinal| self.concept(ordinal))
304            .collect()
305    }
306
307    /// The native codes at `ordinals`, in the order given, `None` where the
308    /// store has no such concept.
309    ///
310    /// Each code is sliced out of the concept column and decoded only as far
311    /// as its code, so the batch costs one string per code and nothing else.
312    /// A caller that can borrow calls [`Self::code`] and pays not even that.
313    ///
314    /// # Errors
315    ///
316    /// Returns [`StoreError`] when a code is damaged.
317    pub fn codes(
318        &self,
319        ordinals: impl IntoIterator<Item = Ordinal>,
320    ) -> Result<Vec<Option<String>>, StoreError> {
321        ordinals
322            .into_iter()
323            .map(|ordinal| Ok(self.code(ordinal)?.map(ToOwned::to_owned)))
324            .collect()
325    }
326
327    /// Every designation of `ordinal`, in index order.
328    ///
329    /// # Errors
330    ///
331    /// Returns [`StoreError`] when the database cannot be read or a record is damaged.
332    pub fn designations(&self, ordinal: Ordinal) -> Result<Vec<Designation>, StoreError> {
333        let Some(packed) = self.designation_rows.get(ordinal.index())? else {
334            return Ok(Vec::new());
335        };
336        record::Designations::decode(packed.value()).map_err(|source| StoreError::Record {
337            table: tables::DESIGNATIONS.name().to_owned(),
338            key: ordinal.to_string(),
339            source,
340        })
341    }
342
343    /// The acceptability ordinal of a designation in a language reference set.
344    ///
345    /// # Errors
346    ///
347    /// Returns [`StoreError`] when the column is damaged.
348    pub fn acceptability(
349        &self,
350        ordinal: Ordinal,
351        designation: u32,
352        language_refset: u32,
353    ) -> Result<Option<u32>, StoreError> {
354        let Some(bytes) = self.acceptability.get(ordinal) else {
355            return Ok(None);
356        };
357        record::Acceptability::find(bytes, designation, language_refset).map_err(|source| {
358            StoreError::Record {
359                table: tables::COLUMN_ACCEPTABILITY.to_owned(),
360                key: ordinal.to_string(),
361                source,
362            }
363        })
364    }
365
366    /// The preferred designation of `ordinal` for a language reference set and
367    /// designation use: the lowest-indexed designation of that use the
368    /// reference set marks preferred.
369    ///
370    /// # Errors
371    ///
372    /// Returns [`StoreError`] when the database cannot be read or a record is
373    /// damaged.
374    pub fn preferred(
375        &self,
376        ordinal: Ordinal,
377        language_refset: u32,
378        use_ordinal: u32,
379    ) -> Result<Option<Designation>, StoreError> {
380        let (Some(preferred), Some(bytes)) = (self.preferred, self.acceptability.get(ordinal))
381        else {
382            return Ok(None);
383        };
384        let entries =
385            record::Acceptability::decode(bytes).map_err(|source| StoreError::Record {
386                table: tables::COLUMN_ACCEPTABILITY.to_owned(),
387                key: ordinal.to_string(),
388                source,
389            })?;
390        let Some(packed) = self.designation_rows.get(ordinal.index())? else {
391            return Ok(None);
392        };
393        let damaged = |source| StoreError::Record {
394            table: tables::DESIGNATIONS.name().to_owned(),
395            key: ordinal.to_string(),
396            source,
397        };
398        let mut chosen: Option<(u32, Designation)> = None;
399        for (index, refset, acceptability) in entries {
400            if refset != language_refset || acceptability != preferred {
401                continue;
402            }
403            if chosen.as_ref().is_some_and(|(at, _)| *at <= index) {
404                continue;
405            }
406            let Some(designation) =
407                record::Designations::at(packed.value(), index).map_err(damaged)?
408            else {
409                continue;
410            };
411            if designation.use_ordinal == use_ordinal {
412                chosen = Some((index, designation));
413            }
414        }
415        Ok(chosen.map(|(_, designation)| designation))
416    }
417
418    /// The display of `ordinal` from the first of `language_refsets` whose
419    /// language `accept` admits.
420    ///
421    /// The build chose one display per reference set, so this reads the column
422    /// and stops at the first language the caller wants; no table is touched.
423    /// The reference sets are asked in the order given, which is what decides
424    /// the answer when several carry the same language.
425    ///
426    /// # Errors
427    ///
428    /// Returns [`StoreError`] when the column is damaged.
429    pub fn display(
430        &self,
431        ordinal: Ordinal,
432        language_refsets: impl IntoIterator<Item = u32>,
433        accept: impl Fn(&str) -> bool,
434    ) -> Result<Option<String>, StoreError> {
435        let Some(bytes) = self.displays.get(ordinal) else {
436            return Ok(None);
437        };
438        let damaged = |source| StoreError::Record {
439            table: tables::COLUMN_DISPLAYS.to_owned(),
440            key: ordinal.to_string(),
441            source,
442        };
443        for refset in language_refsets {
444            let Some((language, term)) = record::Displays::find(bytes, refset).map_err(damaged)?
445            else {
446                continue;
447            };
448            if accept(&language) {
449                return Ok(Some(term));
450            }
451        }
452        Ok(None)
453    }
454
455    /// Every property of `ordinal`, as `(property key ordinal, values)`, in key order.
456    ///
457    /// # Errors
458    ///
459    /// Returns [`StoreError`] when the column is damaged.
460    pub fn properties(
461        &self,
462        ordinal: Ordinal,
463    ) -> Result<Vec<(u32, Vec<PropertyValue>)>, StoreError> {
464        let Some(bytes) = self.properties.get(ordinal) else {
465            return Ok(Vec::new());
466        };
467        record::Properties::decode(bytes).map_err(|source| StoreError::Record {
468            table: tables::COLUMN_PROPERTIES.to_owned(),
469            key: ordinal.to_string(),
470            source,
471        })
472    }
473
474    /// A vocabulary entry: the name of a property key, designation use,
475    /// language reference set, or acceptability ordinal.
476    ///
477    /// # Errors
478    ///
479    /// Returns [`StoreError`] when the database cannot be read.
480    pub fn vocabulary(&self, kind: Vocabulary, ordinal: u32) -> Result<Option<String>, StoreError> {
481        let txn = self.db.begin_read()?;
482        let table = open_table!(txn, kind.table())?;
483        Ok(table.get(ordinal)?.map(|v| v.value().to_owned()))
484    }
485
486    /// The ordinal of a vocabulary name, by scanning the (small) vocabulary table.
487    ///
488    /// # Errors
489    ///
490    /// Returns [`StoreError`] when the database cannot be read.
491    pub fn vocabulary_ordinal(
492        &self,
493        kind: Vocabulary,
494        name: &str,
495    ) -> Result<Option<u32>, StoreError> {
496        let txn = self.db.begin_read()?;
497        let table = open_table!(txn, kind.table())?;
498        for entry in table.iter()? {
499            let (key, value) = entry?;
500            if value.value() == name {
501                return Ok(Some(key.value()));
502            }
503        }
504        Ok(None)
505    }
506}
507
508/// The small name tables of an artifact.
509#[derive(Debug, Clone, Copy, PartialEq, Eq)]
510pub enum Vocabulary {
511    /// Property key names.
512    PropertyKeys,
513    /// Designation use codes.
514    DesignationUses,
515    /// Language reference set codes.
516    LanguageRefsets,
517    /// Acceptability codes.
518    Acceptabilities,
519}
520
521impl Vocabulary {
522    fn table(self) -> redb::TableDefinition<'static, u32, &'static str> {
523        match self {
524            Self::PropertyKeys => tables::PROPERTY_KEYS,
525            Self::DesignationUses => tables::DESIGNATION_USES,
526            Self::LanguageRefsets => tables::LANGUAGE_REFSETS,
527            Self::Acceptabilities => tables::ACCEPTABILITIES,
528        }
529    }
530}
concept_store/store.rs

concept_store/
store.rs
