concept_store/
builder.rs

1//! Writing an artifact, offline, in one transaction.
2//!
3//! The build tool feeds concepts, designations, acceptability, properties, and
4//! nothing else; `finish` packs each ordinal-keyed set into a dense column,
5//! chooses the displays, and commits. Two builds from the same input produce
6//! the same bytes.
7
8use std::collections::BTreeMap;
9use std::path::{Path, PathBuf};
10
11use concept_graph::ordinal::Ordinal;
12use redb::{Database, TableHandle};
13
14use crate::column::Column;
15use crate::record::{self, Concept, Designation, PropertyValue};
16use crate::tables;
17
18/// A failure while building.
19#[derive(Debug, thiserror::Error)]
20pub enum BuildError {
21    /// The database could not be created.
22    #[error("cannot create the artifact at {path}")]
23    Create {
24        /// The artifact.
25        path: PathBuf,
26        /// The underlying error.
27        #[source]
28        source: redb::DatabaseError,
29    },
30    /// A transaction failed.
31    #[error("transaction failed")]
32    Transaction(#[from] redb::TransactionError),
33    /// A table could not be opened.
34    #[error("cannot open table")]
35    Table(#[from] redb::TableError),
36    /// A write failed inside the database.
37    #[error("storage write failed")]
38    Storage(#[from] redb::StorageError),
39    /// The commit failed.
40    #[error("commit failed")]
41    Commit(#[from] redb::CommitError),
42    /// Reclaiming the unused pages after the commit failed.
43    #[error("compaction failed")]
44    Compaction(#[from] redb::CompactionError),
45    /// A vocabulary name was registered twice with different ordinals.
46    #[error("{kind} {name:?} is already ordinal {existing}, not {requested}")]
47    Vocabulary {
48        /// The vocabulary.
49        kind: String,
50        /// The name.
51        name: String,
52        /// The ordinal it already has.
53        existing: u32,
54        /// The ordinal requested.
55        requested: u32,
56    },
57}
58
59/// The acceptability that marks a designation as preferred in a language
60/// reference set, as the code system spells it (for SNOMED, the SCTID of
61/// `|Preferred|`).
62#[derive(Debug, Clone, PartialEq, Eq)]
63pub struct PreferredRule {
64    /// The acceptability ordinal meaning preferred.
65    pub preferred: u32,
66    /// The designation use whose preferred designation is a concept display
67    /// (for SNOMED, the synonym), or `None` when the system has no such use.
68    ///
69    /// Only this use goes into the display column. Carrying every use made the
70    /// column a second copy of most of the designations (#322).
71    pub display_use: Option<u32>,
72}
73
74/// An artifact under construction.
75///
76/// Rows are buffered in memory, sorted by key, and written in `finish`, each
77/// table opened once. redb fills pages tightly when keys arrive in order, and
78/// opening a table is not free; the first build wrote every row through its
79/// own `open_table` in arrival order and took ten minutes for the NL edition.
80pub struct StoreBuilder {
81    path: PathBuf,
82    db: Database,
83    system: String,
84    version: String,
85    vocabularies: BTreeMap<&'static str, BTreeMap<u32, String>>,
86    codes: BTreeMap<String, u32>,
87    concepts: BTreeMap<u32, Vec<u8>>,
88    designations: BTreeMap<(u32, u32), Vec<u8>>,
89    acceptability: BTreeMap<(u32, u32, u32), u32>,
90    designation_uses: BTreeMap<(u32, u32), u32>,
91    properties: BTreeMap<(u32, u32), Vec<u8>>,
92}
93
94impl std::fmt::Debug for StoreBuilder {
95    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
96        f.debug_struct("StoreBuilder")
97            .field("path", &self.path)
98            .field("concepts", &self.concepts.len())
99            .field("designations", &self.designations.len())
100            .finish_non_exhaustive()
101    }
102}
103
104impl StoreBuilder {
105    /// Creates the artifact at `path`, replacing any file there, and records
106    /// the layout version, system, and version.
107    ///
108    /// # Errors
109    ///
110    /// Returns [`BuildError`] when the file cannot be created.
111    pub fn create(path: &Path, system: &str, version: &str) -> Result<Self, BuildError> {
112        if path.exists() {
113            std::fs::remove_file(path).map_err(|source| BuildError::Create {
114                path: path.to_path_buf(),
115                source: redb::DatabaseError::Storage(redb::StorageError::Io(source)),
116            })?;
117        }
118        let db = Database::create(path).map_err(|source| BuildError::Create {
119            path: path.to_path_buf(),
120            source,
121        })?;
122        Ok(Self {
123            path: path.to_path_buf(),
124            db,
125            system: system.to_owned(),
126            version: version.to_owned(),
127            vocabularies: BTreeMap::new(),
128            codes: BTreeMap::new(),
129            concepts: BTreeMap::new(),
130            designations: BTreeMap::new(),
131            acceptability: BTreeMap::new(),
132            designation_uses: BTreeMap::new(),
133            properties: BTreeMap::new(),
134        })
135    }
136
137    /// Names a vocabulary ordinal (a property key, designation use, language
138    /// reference set, or acceptability).
139    ///
140    /// # Errors
141    ///
142    /// Returns [`BuildError::Vocabulary`] when `name` already has another ordinal.
143    pub fn vocabulary(
144        &mut self,
145        kind: crate::store::Vocabulary,
146        ordinal: u32,
147        name: &str,
148    ) -> Result<(), BuildError> {
149        let table_name = match kind {
150            crate::store::Vocabulary::PropertyKeys => tables::PROPERTY_KEYS.name(),
151            crate::store::Vocabulary::DesignationUses => tables::DESIGNATION_USES.name(),
152            crate::store::Vocabulary::LanguageRefsets => tables::LANGUAGE_REFSETS.name(),
153            crate::store::Vocabulary::Acceptabilities => tables::ACCEPTABILITIES.name(),
154        };
155        let entries = self.vocabularies.entry(table_name).or_default();
156        if let Some((existing, _)) = entries
157            .iter()
158            .find(|(key, value)| value.as_str() == name && **key != ordinal)
159        {
160            return Err(BuildError::Vocabulary {
161                kind: table_name.to_owned(),
162                name: name.to_owned(),
163                existing: *existing,
164                requested: ordinal,
165            });
166        }
167        entries.insert(ordinal, name.to_owned());
168        Ok(())
169    }
170
171    /// Adds a concept under `ordinal`.
172    ///
173    /// # Errors
174    ///
175    /// Cannot fail today; the signature keeps the door open for a full buffer.
176    #[expect(
177        clippy::unnecessary_wraps,
178        reason = "the build seam is fallible by contract: a full buffer fails here"
179    )]
180    pub fn concept(&mut self, ordinal: Ordinal, concept: &Concept) -> Result<(), BuildError> {
181        self.codes.insert(concept.code.clone(), ordinal.index());
182        self.concepts.insert(ordinal.index(), concept.encode());
183        Ok(())
184    }
185
186    /// Adds designation `index` of concept `ordinal`.
187    ///
188    /// # Errors
189    ///
190    /// Cannot fail today; the signature keeps the door open for a full buffer.
191    #[expect(
192        clippy::unnecessary_wraps,
193        reason = "the build seam is fallible by contract: a full buffer fails here"
194    )]
195    pub fn designation(
196        &mut self,
197        ordinal: Ordinal,
198        index: u32,
199        designation: &Designation,
200    ) -> Result<(), BuildError> {
201        self.designations
202            .insert((ordinal.index(), index), designation.encode());
203        self.designation_uses
204            .insert((ordinal.index(), index), designation.use_ordinal);
205        Ok(())
206    }
207
208    /// Records the acceptability of a designation in a language reference set.
209    ///
210    /// # Errors
211    ///
212    /// Cannot fail today; the signature keeps the door open for a full buffer.
213    #[expect(
214        clippy::unnecessary_wraps,
215        reason = "the build seam is fallible by contract: a full buffer fails here"
216    )]
217    pub fn acceptability(
218        &mut self,
219        ordinal: Ordinal,
220        index: u32,
221        language_refset: u32,
222        acceptability: u32,
223    ) -> Result<(), BuildError> {
224        self.acceptability
225            .insert((ordinal.index(), index, language_refset), acceptability);
226        Ok(())
227    }
228
229    /// Sets the values of property `key` on concept `ordinal`.
230    ///
231    /// # Errors
232    ///
233    /// Cannot fail today; the signature keeps the door open for a full buffer.
234    #[expect(
235        clippy::unnecessary_wraps,
236        reason = "the build seam is fallible by contract: a full buffer fails here"
237    )]
238    pub fn properties(
239        &mut self,
240        ordinal: Ordinal,
241        key: u32,
242        values: &[PropertyValue],
243    ) -> Result<(), BuildError> {
244        self.properties
245            .insert((ordinal.index(), key), PropertyValue::encode_list(values));
246        Ok(())
247    }
248
249    /// The preferred designation index per `(concept, reference set, use)`:
250    /// the lowest index among those the reference set marks preferred.
251    fn chosen_preferred(&self, rule: &PreferredRule) -> BTreeMap<(u32, u32, u32), u32> {
252        let mut chosen: BTreeMap<(u32, u32, u32), u32> = BTreeMap::new();
253        for ((concept, index, refset), acceptability) in &self.acceptability {
254            if *acceptability != rule.preferred {
255                continue;
256            }
257            let Some(use_ordinal) = self.designation_uses.get(&(*concept, *index)) else {
258                continue;
259            };
260            chosen
261                .entry((*concept, *refset, *use_ordinal))
262                .and_modify(|existing| *existing = (*existing).min(*index))
263                .or_insert(*index);
264        }
265        chosen
266    }
267
268    /// The chosen designations themselves, packed by concept.
269    ///
270    /// Choosing a display is the hottest read the server has, and it should
271    /// not have to find the designation this build already found.
272    fn display_column(
273        &self,
274        chosen: &BTreeMap<(u32, u32, u32), u32>,
275        display_use: u32,
276        count: u32,
277    ) -> Vec<u8> {
278        let mut by_concept: BTreeMap<u32, Vec<(u32, String, String)>> = BTreeMap::new();
279        for ((concept, refset, use_ordinal), index) in chosen {
280            if *use_ordinal != display_use {
281                continue;
282            }
283            let Some(bytes) = self.designations.get(&(*concept, *index)) else {
284                continue;
285            };
286            let Ok(designation) = Designation::decode(bytes) else {
287                continue;
288            };
289            by_concept.entry(*concept).or_default().push((
290                *refset,
291                designation.language,
292                designation.term,
293            ));
294        }
295        let packed: BTreeMap<u32, Vec<u8>> = by_concept
296            .into_iter()
297            .map(|(concept, entries)| {
298                let borrowed: Vec<(u32, &str, &str)> = entries
299                    .iter()
300                    .map(|(refset, language, term)| (*refset, language.as_str(), term.as_str()))
301                    .collect();
302                (concept, record::Displays::encode(&borrowed))
303            })
304            .collect();
305        Column::pack(
306            count,
307            packed
308                .iter()
309                .map(|(concept, bytes)| (Ordinal::new(*concept), bytes.as_slice())),
310        )
311    }
312
313    /// Calls `write` once per concept with that concept's rows.
314    ///
315    /// The rows arrive keyed by a concept ordinal followed by the rest of the
316    /// key, so a single pass in key order groups them without sorting again.
317    /// `write` takes the group rather than a finished record so a caller that
318    /// streams to a table holds no second copy of the edition.
319    fn per_concept<'a, K: 'a, V: 'a, R>(
320        rows: &'a BTreeMap<K, V>,
321        concept: impl Fn(&K) -> u32,
322        row: impl Fn(&'a K, &'a V) -> R,
323        mut write: impl FnMut(u32, &[R]) -> Result<(), BuildError>,
324    ) -> Result<(), BuildError> {
325        let mut at = None;
326        let mut group: Vec<R> = Vec::new();
327        for (key, value) in rows {
328            let ordinal = concept(key);
329            if at != Some(ordinal) {
330                if let Some(previous) = at {
331                    write(previous, &group)?;
332                }
333                at = Some(ordinal);
334                group.clear();
335            }
336            group.push(row(key, value));
337        }
338        match at {
339            Some(previous) => write(previous, &group),
340            None => Ok(()),
341        }
342    }
343
344    /// The dense column of one record per concept, packed by `pack`.
345    fn column<'a, K: 'a, V: 'a, R>(
346        count: u32,
347        rows: &'a BTreeMap<K, V>,
348        concept: impl Fn(&K) -> u32,
349        row: impl Fn(&'a K, &'a V) -> R,
350        pack: impl Fn(&[R]) -> Vec<u8>,
351    ) -> Result<Vec<u8>, BuildError> {
352        let mut packed: BTreeMap<u32, Vec<u8>> = BTreeMap::new();
353        Self::per_concept(rows, concept, row, |ordinal, group| {
354            packed.insert(ordinal, pack(group));
355            Ok(())
356        })?;
357        Ok(Column::pack(
358            count,
359            packed
360                .iter()
361                .map(|(ordinal, bytes)| (Ordinal::new(*ordinal), bytes.as_slice())),
362        ))
363    }
364
365    /// Writes every buffered row in key order, computes the preferred
366    /// designations per language reference set and use, records the concept
367    /// count, and commits.
368    ///
369    /// # Errors
370    ///
371    /// Returns [`BuildError`] when a table cannot be written or the commit fails.
372    pub fn finish(self, rule: &PreferredRule) -> Result<PathBuf, BuildError> {
373        let txn = self.db.begin_write()?;
374        {
375            let mut meta = txn.open_table(tables::META)?;
376            meta.insert(tables::META_LAYOUT, tables::LAYOUT_VERSION)?;
377            meta.insert(tables::META_SYSTEM, self.system.as_str())?;
378            meta.insert(tables::META_VERSION, self.version.as_str())?;
379            meta.insert(
380                tables::META_CONCEPTS,
381                self.concepts.len().to_string().as_str(),
382            )?;
383            meta.insert(tables::META_PREFERRED, rule.preferred.to_string().as_str())?;
384        }
385        {
386            let mut codes = txn.open_table(tables::CODES)?;
387            for (code, ordinal) in &self.codes {
388                codes.insert(code.as_str(), *ordinal)?;
389            }
390        }
391        // The count is the highest ordinal, not the entry count, so a gap in
392        // the ordinals cannot push a record off the end of a column.
393        let count = self
394            .concepts
395            .keys()
396            .next_back()
397            .map_or(0, |highest| highest.saturating_add(1));
398        {
399            let mut columns = txn.open_table(tables::COLUMNS)?;
400            // An ordinal is a position, so these are dense columns rather
401            // than b-trees keyed by that position.
402            let concepts = Column::pack(
403                count,
404                self.concepts
405                    .iter()
406                    .map(|(ordinal, bytes)| (Ordinal::new(*ordinal), bytes.as_slice())),
407            );
408            columns.insert(tables::COLUMN_CONCEPTS, concepts.as_slice())?;
409            let acceptability = Self::column(
410                count,
411                &self.acceptability,
412                |(concept, _, _)| *concept,
413                |(_, index, refset), acceptability| (*index, *refset, *acceptability),
414                record::Acceptability::encode,
415            )?;
416            columns.insert(tables::COLUMN_ACCEPTABILITY, acceptability.as_slice())?;
417            let properties = Self::column(
418                count,
419                &self.properties,
420                |(concept, _)| *concept,
421                |(_, key), values| (*key, values.as_slice()),
422                record::Properties::encode,
423            )?;
424            columns.insert(tables::COLUMN_PROPERTIES, properties.as_slice())?;
425        }
426        self.write_designations(&txn)?;
427        for table_def in [
428            tables::PROPERTY_KEYS,
429            tables::DESIGNATION_USES,
430            tables::LANGUAGE_REFSETS,
431            tables::ACCEPTABILITIES,
432        ] {
433            let mut table = txn.open_table(table_def)?;
434            if let Some(entries) = self.vocabularies.get(table_def.name()) {
435                for (ordinal, name) in entries {
436                    table.insert(*ordinal, name.as_str())?;
437                }
438            }
439        }
440        if let Some(display_use) = rule.display_use {
441            let chosen = self.chosen_preferred(rule);
442            let displays = self.display_column(&chosen, display_use, count);
443            let mut columns = txn.open_table(tables::COLUMNS)?;
444            columns.insert(tables::COLUMN_DISPLAYS, displays.as_slice())?;
445        }
446        txn.commit()?;
447        // redb grows the file in regions ahead of use; compaction returns the
448        // unused pages so the artifact is the size of its data. Repeated until
449        // redb reports nothing further to reclaim.
450        let mut db = self.db;
451        while db.compact()? {}
452        drop(db);
453        Ok(self.path)
454    }
455
456    /// Writes the designations of each concept as one row.
457    ///
458    /// The designation text is the largest thing the artifact holds, so it
459    /// stays in the database; one row per concept keys the tree by what a
460    /// reader asks for and holds no key per designation (#338).
461    fn write_designations(&self, txn: &redb::WriteTransaction) -> Result<(), BuildError> {
462        let mut designations = txn.open_table(tables::DESIGNATIONS)?;
463        Self::per_concept(
464            &self.designations,
465            |(concept, _)| *concept,
466            |(_, index), bytes| (*index, bytes.as_slice()),
467            |ordinal, group| {
468                designations.insert(ordinal, record::Designations::encode(group).as_slice())?;
469                Ok(())
470            },
471        )
472    }
473}
concept_store/builder.rs

concept_store/
builder.rs
