concept_store/
builder.rs

1//! Writing an artifact, offline, in one transaction.
2//!
3//! The build tool feeds concepts, designations, acceptability, properties, and
4//! nothing else; `finish` packs each ordinal-keyed set into a dense column
5//! beside the database, chooses the displays, and commits. Two builds from the
6//! same input produce the same bytes.
7
8use std::collections::BTreeMap;
9use std::io::{BufWriter, Write};
10use std::path::{Path, PathBuf};
11
12use concept_graph::ordinal::Ordinal;
13use redb::{Database, TableHandle};
14
15use crate::column::Column;
16use crate::record::{self, Concept, Designation, PropertyValue};
17use crate::tables;
18
19/// How much of a column's side file the writer holds before it goes to disk.
20///
21/// A column is written as a count, an offset per record, and the records, each
22/// a small write, so the buffer is what turns them into whole-page writes.
23const WRITE_BUFFER: usize = 1 << 20;
24
25/// A failure while building.
26#[derive(Debug, thiserror::Error)]
27pub enum BuildError {
28    /// The database could not be created.
29    #[error("cannot create the artifact at {path}")]
30    Create {
31        /// The artifact.
32        path: PathBuf,
33        /// The underlying error.
34        #[source]
35        source: redb::DatabaseError,
36    },
37    /// A transaction failed.
38    #[error("transaction failed")]
39    Transaction(#[from] redb::TransactionError),
40    /// A table could not be opened.
41    #[error("cannot open table")]
42    Table(#[from] redb::TableError),
43    /// A write failed inside the database.
44    #[error("storage write failed")]
45    Storage(#[from] redb::StorageError),
46    /// The commit failed.
47    #[error("commit failed")]
48    Commit(#[from] redb::CommitError),
49    /// Reclaiming the unused pages after the commit failed.
50    #[error("compaction failed")]
51    Compaction(#[from] redb::CompactionError),
52    /// A column's side file could not be written.
53    #[error("cannot write {path}")]
54    Io {
55        /// The file.
56        path: PathBuf,
57        /// The underlying error.
58        #[source]
59        source: std::io::Error,
60    },
61    /// A vocabulary name was registered twice with different ordinals.
62    #[error("{kind} {name:?} is already ordinal {existing}, not {requested}")]
63    Vocabulary {
64        /// The vocabulary.
65        kind: String,
66        /// The name.
67        name: String,
68        /// The ordinal it already has.
69        existing: u32,
70        /// The ordinal requested.
71        requested: u32,
72    },
73}
74
75/// The acceptability that marks a designation as preferred in a language
76/// reference set, as the code system spells it (for SNOMED, the SCTID of
77/// `|Preferred|`).
78#[derive(Debug, Clone, PartialEq, Eq)]
79pub struct PreferredRule {
80    /// The acceptability ordinal meaning preferred.
81    pub preferred: u32,
82    /// The designation use whose preferred designation is a concept display
83    /// (for SNOMED, the synonym), or `None` when the system has no such use.
84    ///
85    /// Only this use goes into the display column. Carrying every use made the
86    /// column a second copy of most of the designations (#322).
87    pub display_use: Option<u32>,
88}
89
90/// An artifact under construction.
91///
92/// Rows are buffered in memory, sorted by key, and written in `finish`, each
93/// table opened once. redb fills pages tightly when keys arrive in order, and
94/// opening a table is not free; the first build wrote every row through its
95/// own `open_table` in arrival order and took ten minutes for the NL edition.
96pub struct StoreBuilder {
97    path: PathBuf,
98    db: Database,
99    system: String,
100    version: String,
101    vocabularies: BTreeMap<&'static str, BTreeMap<u32, String>>,
102    codes: BTreeMap<String, u32>,
103    concepts: BTreeMap<u32, Vec<u8>>,
104    designations: BTreeMap<(u32, u32), Vec<u8>>,
105    acceptability: BTreeMap<(u32, u32, u32), u32>,
106    designation_uses: BTreeMap<(u32, u32), u32>,
107    properties: BTreeMap<(u32, u32), Vec<u8>>,
108}
109
110impl std::fmt::Debug for StoreBuilder {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        f.debug_struct("StoreBuilder")
113            .field("path", &self.path)
114            .field("concepts", &self.concepts.len())
115            .field("designations", &self.designations.len())
116            .finish_non_exhaustive()
117    }
118}
119
120impl StoreBuilder {
121    /// Creates the artifact at `path`, replacing any file there, and records
122    /// the layout version, system, and version.
123    ///
124    /// # Errors
125    ///
126    /// Returns [`BuildError`] when the file cannot be created.
127    pub fn create(path: &Path, system: &str, version: &str) -> Result<Self, BuildError> {
128        if path.exists() {
129            std::fs::remove_file(path).map_err(|source| BuildError::Create {
130                path: path.to_path_buf(),
131                source: redb::DatabaseError::Storage(redb::StorageError::Io(source)),
132            })?;
133        }
134        let db = Database::create(path).map_err(|source| BuildError::Create {
135            path: path.to_path_buf(),
136            source,
137        })?;
138        Ok(Self {
139            path: path.to_path_buf(),
140            db,
141            system: system.to_owned(),
142            version: version.to_owned(),
143            vocabularies: BTreeMap::new(),
144            codes: BTreeMap::new(),
145            concepts: BTreeMap::new(),
146            designations: BTreeMap::new(),
147            acceptability: BTreeMap::new(),
148            designation_uses: BTreeMap::new(),
149            properties: BTreeMap::new(),
150        })
151    }
152
153    /// Names a vocabulary ordinal (a property key, designation use, language
154    /// reference set, or acceptability).
155    ///
156    /// # Errors
157    ///
158    /// Returns [`BuildError::Vocabulary`] when `name` already has another ordinal.
159    pub fn vocabulary(
160        &mut self,
161        kind: crate::store::Vocabulary,
162        ordinal: u32,
163        name: &str,
164    ) -> Result<(), BuildError> {
165        let table_name = match kind {
166            crate::store::Vocabulary::PropertyKeys => tables::PROPERTY_KEYS.name(),
167            crate::store::Vocabulary::DesignationUses => tables::DESIGNATION_USES.name(),
168            crate::store::Vocabulary::LanguageRefsets => tables::LANGUAGE_REFSETS.name(),
169            crate::store::Vocabulary::Acceptabilities => tables::ACCEPTABILITIES.name(),
170        };
171        let entries = self.vocabularies.entry(table_name).or_default();
172        if let Some((existing, _)) = entries
173            .iter()
174            .find(|(key, value)| value.as_str() == name && **key != ordinal)
175        {
176            return Err(BuildError::Vocabulary {
177                kind: table_name.to_owned(),
178                name: name.to_owned(),
179                existing: *existing,
180                requested: ordinal,
181            });
182        }
183        entries.insert(ordinal, name.to_owned());
184        Ok(())
185    }
186
187    /// Adds a concept under `ordinal`.
188    ///
189    /// # Errors
190    ///
191    /// Cannot fail today; the signature keeps the door open for a full buffer.
192    #[expect(
193        clippy::unnecessary_wraps,
194        reason = "the build seam is fallible by contract: a full buffer fails here"
195    )]
196    pub fn concept(&mut self, ordinal: Ordinal, concept: &Concept) -> Result<(), BuildError> {
197        self.codes.insert(concept.code.clone(), ordinal.index());
198        self.concepts.insert(ordinal.index(), concept.encode());
199        Ok(())
200    }
201
202    /// Adds designation `index` of concept `ordinal`.
203    ///
204    /// # Errors
205    ///
206    /// Cannot fail today; the signature keeps the door open for a full buffer.
207    #[expect(
208        clippy::unnecessary_wraps,
209        reason = "the build seam is fallible by contract: a full buffer fails here"
210    )]
211    pub fn designation(
212        &mut self,
213        ordinal: Ordinal,
214        index: u32,
215        designation: &Designation,
216    ) -> Result<(), BuildError> {
217        self.designations
218            .insert((ordinal.index(), index), designation.encode());
219        self.designation_uses
220            .insert((ordinal.index(), index), designation.use_ordinal);
221        Ok(())
222    }
223
224    /// Records the acceptability of a designation in a language reference set.
225    ///
226    /// # Errors
227    ///
228    /// Cannot fail today; the signature keeps the door open for a full buffer.
229    #[expect(
230        clippy::unnecessary_wraps,
231        reason = "the build seam is fallible by contract: a full buffer fails here"
232    )]
233    pub fn acceptability(
234        &mut self,
235        ordinal: Ordinal,
236        index: u32,
237        language_refset: u32,
238        acceptability: u32,
239    ) -> Result<(), BuildError> {
240        self.acceptability
241            .insert((ordinal.index(), index, language_refset), acceptability);
242        Ok(())
243    }
244
245    /// Sets the values of property `key` on concept `ordinal`.
246    ///
247    /// # Errors
248    ///
249    /// Cannot fail today; the signature keeps the door open for a full buffer.
250    #[expect(
251        clippy::unnecessary_wraps,
252        reason = "the build seam is fallible by contract: a full buffer fails here"
253    )]
254    pub fn properties(
255        &mut self,
256        ordinal: Ordinal,
257        key: u32,
258        values: &[PropertyValue],
259    ) -> Result<(), BuildError> {
260        self.properties
261            .insert((ordinal.index(), key), PropertyValue::encode_list(values));
262        Ok(())
263    }
264
265    /// The preferred designation index per `(concept, reference set, use)`:
266    /// the lowest index among those the reference set marks preferred.
267    fn chosen_preferred(&self, rule: &PreferredRule) -> BTreeMap<(u32, u32, u32), u32> {
268        let mut chosen: BTreeMap<(u32, u32, u32), u32> = BTreeMap::new();
269        for ((concept, index, refset), acceptability) in &self.acceptability {
270            if *acceptability != rule.preferred {
271                continue;
272            }
273            let Some(use_ordinal) = self.designation_uses.get(&(*concept, *index)) else {
274                continue;
275            };
276            chosen
277                .entry((*concept, *refset, *use_ordinal))
278                .and_modify(|existing| *existing = (*existing).min(*index))
279                .or_insert(*index);
280        }
281        chosen
282    }
283
284    /// The chosen designations themselves, packed by concept.
285    ///
286    /// Choosing a display is the hottest read the server has, and it should
287    /// not have to find the designation this build already found.
288    fn display_rows(
289        &self,
290        chosen: &BTreeMap<(u32, u32, u32), u32>,
291        display_use: u32,
292    ) -> BTreeMap<u32, Vec<u8>> {
293        let mut by_concept: BTreeMap<u32, Vec<(u32, String, String)>> = BTreeMap::new();
294        for ((concept, refset, use_ordinal), index) in chosen {
295            if *use_ordinal != display_use {
296                continue;
297            }
298            let Some(bytes) = self.designations.get(&(*concept, *index)) else {
299                continue;
300            };
301            let Ok(designation) = Designation::decode(bytes) else {
302                continue;
303            };
304            by_concept.entry(*concept).or_default().push((
305                *refset,
306                designation.language,
307                designation.term,
308            ));
309        }
310        by_concept
311            .into_iter()
312            .map(|(concept, entries)| {
313                let borrowed: Vec<(u32, &str, &str)> = entries
314                    .iter()
315                    .map(|(refset, language, term)| (*refset, language.as_str(), term.as_str()))
316                    .collect();
317                (concept, record::Displays::encode(&borrowed))
318            })
319            .collect()
320    }
321
322    /// Calls `write` once per concept with that concept's rows.
323    ///
324    /// The rows arrive keyed by a concept ordinal followed by the rest of the
325    /// key, so a single pass in key order groups them without sorting again.
326    /// `write` takes the group rather than a finished record so a caller that
327    /// streams to a table holds no second copy of the edition.
328    fn per_concept<'a, K: 'a, V: 'a, R>(
329        rows: &'a BTreeMap<K, V>,
330        concept: impl Fn(&K) -> u32,
331        row: impl Fn(&'a K, &'a V) -> R,
332        mut write: impl FnMut(u32, &[R]) -> Result<(), BuildError>,
333    ) -> Result<(), BuildError> {
334        let mut at = None;
335        let mut group: Vec<R> = Vec::new();
336        for (key, value) in rows {
337            let ordinal = concept(key);
338            if at != Some(ordinal) {
339                if let Some(previous) = at {
340                    write(previous, &group)?;
341                }
342                at = Some(ordinal);
343                group.clear();
344            }
345            group.push(row(key, value));
346        }
347        match at {
348            Some(previous) => write(previous, &group),
349            None => Ok(()),
350        }
351    }
352
353    /// One record per concept, encoded by `pack`, keyed by concept ordinal.
354    fn column_rows<'a, K: 'a, V: 'a, R>(
355        rows: &'a BTreeMap<K, V>,
356        concept: impl Fn(&K) -> u32,
357        row: impl Fn(&'a K, &'a V) -> R,
358        pack: impl Fn(&[R]) -> Vec<u8>,
359    ) -> Result<BTreeMap<u32, Vec<u8>>, BuildError> {
360        let mut packed: BTreeMap<u32, Vec<u8>> = BTreeMap::new();
361        Self::per_concept(rows, concept, row, |ordinal, group| {
362            packed.insert(ordinal, pack(group));
363            Ok(())
364        })?;
365        Ok(packed)
366    }
367
368    /// Writes one dense column to its own file beside the database.
369    ///
370    /// A column is a position-addressed array, which a b-tree of positions
371    /// only pretends to be, and a reader of a file assembles it at the address
372    /// it serves from (#641).
373    fn write_column(
374        &self,
375        name: &str,
376        count: u32,
377        rows: &BTreeMap<u32, Vec<u8>>,
378    ) -> Result<(), BuildError> {
379        let path = Column::file(&self.path, name);
380        let failed = |source| BuildError::Io {
381            path: path.clone(),
382            source,
383        };
384        let file = std::fs::File::create(&path).map_err(failed)?;
385        let mut out = BufWriter::with_capacity(WRITE_BUFFER, file);
386        Column::write_to(
387            count,
388            rows.iter()
389                .map(|(ordinal, bytes)| (Ordinal::new(*ordinal), bytes.as_slice())),
390            &mut out,
391        )
392        .map_err(failed)?;
393        out.flush().map_err(failed)
394    }
395
396    /// Writes every buffered row in key order, computes the preferred
397    /// designations per language reference set and use, records the concept
398    /// count, and commits.
399    ///
400    /// # Errors
401    ///
402    /// Returns [`BuildError`] when a table cannot be written or the commit fails.
403    pub fn finish(self, rule: &PreferredRule) -> Result<PathBuf, BuildError> {
404        let txn = self.db.begin_write()?;
405        {
406            let mut meta = txn.open_table(tables::META)?;
407            meta.insert(tables::META_LAYOUT, tables::LAYOUT_VERSION)?;
408            meta.insert(tables::META_SYSTEM, self.system.as_str())?;
409            meta.insert(tables::META_VERSION, self.version.as_str())?;
410            meta.insert(
411                tables::META_CONCEPTS,
412                self.concepts.len().to_string().as_str(),
413            )?;
414            meta.insert(tables::META_PREFERRED, rule.preferred.to_string().as_str())?;
415        }
416        {
417            let mut codes = txn.open_table(tables::CODES)?;
418            for (code, ordinal) in &self.codes {
419                codes.insert(code.as_str(), *ordinal)?;
420            }
421        }
422        // The count is the highest ordinal, not the entry count, so a gap in
423        // the ordinals cannot push a record off the end of a column.
424        let count = self
425            .concepts
426            .keys()
427            .next_back()
428            .map_or(0, |highest| highest.saturating_add(1));
429        // An ordinal is a position, so these are dense columns in their own
430        // files rather than b-trees keyed by that position.
431        self.write_column(tables::COLUMN_CONCEPTS, count, &self.concepts)?;
432        self.write_column(
433            tables::COLUMN_ACCEPTABILITY,
434            count,
435            &Self::column_rows(
436                &self.acceptability,
437                |(concept, _, _)| *concept,
438                |(_, index, refset), acceptability| (*index, *refset, *acceptability),
439                record::Acceptability::encode,
440            )?,
441        )?;
442        self.write_column(
443            tables::COLUMN_PROPERTIES,
444            count,
445            &Self::column_rows(
446                &self.properties,
447                |(concept, _)| *concept,
448                |(_, key), values| (*key, values.as_slice()),
449                record::Properties::encode,
450            )?,
451        )?;
452        // A code system with no display use still gets the file, empty, so a
453        // store that opens one reads four columns or refuses.
454        let displays = rule.display_use.map_or_else(BTreeMap::new, |display_use| {
455            self.display_rows(&self.chosen_preferred(rule), display_use)
456        });
457        self.write_column(tables::COLUMN_DISPLAYS, count, &displays)?;
458        self.write_designations(&txn)?;
459        for table_def in [
460            tables::PROPERTY_KEYS,
461            tables::DESIGNATION_USES,
462            tables::LANGUAGE_REFSETS,
463            tables::ACCEPTABILITIES,
464        ] {
465            let mut table = txn.open_table(table_def)?;
466            if let Some(entries) = self.vocabularies.get(table_def.name()) {
467                for (ordinal, name) in entries {
468                    table.insert(*ordinal, name.as_str())?;
469                }
470            }
471        }
472        txn.commit()?;
473        // redb grows the file in regions ahead of use; compaction returns the
474        // unused pages so the artifact is the size of its data. Repeated until
475        // redb reports nothing further to reclaim.
476        let mut db = self.db;
477        while db.compact()? {}
478        drop(db);
479        Ok(self.path)
480    }
481
482    /// Writes the designations of each concept as one row.
483    ///
484    /// The designation text is the largest thing the artifact holds, so it
485    /// stays in the database; one row per concept keys the tree by what a
486    /// reader asks for and holds no key per designation (#338).
487    fn write_designations(&self, txn: &redb::WriteTransaction) -> Result<(), BuildError> {
488        let mut designations = txn.open_table(tables::DESIGNATIONS)?;
489        Self::per_concept(
490            &self.designations,
491            |(concept, _)| *concept,
492            |(_, index), bytes| (*index, bytes.as_slice()),
493            |ordinal, group| {
494                designations.insert(ordinal, record::Designations::encode(group).as_slice())?;
495                Ok(())
496            },
497        )
498    }
499}
concept_store/builder.rs

concept_store/
builder.rs
