concept_store/
builder.rs

1// SPDX-FileCopyrightText: Cadasto B.V.
2// SPDX-License-Identifier: BUSL-1.1
3
4//! Writing an artifact, offline, in one transaction.
5//!
6//! The build tool feeds concepts, designations, acceptability, properties, and
7//! nothing else; `finish` packs each ordinal-keyed set into a dense column
8//! beside the database, chooses the displays, and commits. Two builds from the
9//! same input produce the same bytes.
10
11use std::collections::BTreeMap;
12use std::io::{BufWriter, Write};
13use std::path::{Path, PathBuf};
14
15use concept_graph::ordinal::Ordinal;
16use redb::{Database, TableHandle};
17
18use crate::column::Column;
19use crate::record::{self, Concept, Designation, PropertyValue};
20use crate::tables;
21
22/// How much of a column's side file the writer holds before it goes to disk.
23///
24/// A column is written as a count, an offset per record, and the records, each
25/// a small write, so the buffer is what turns them into whole-page writes.
26const WRITE_BUFFER: usize = 1 << 20;
27
28/// A failure while building.
29#[derive(Debug, thiserror::Error)]
30pub enum BuildError {
31    /// The database could not be created.
32    #[error("cannot create the artifact at {path}")]
33    Create {
34        /// The artifact.
35        path: PathBuf,
36        /// The underlying error.
37        #[source]
38        source: redb::DatabaseError,
39    },
40    /// A transaction failed.
41    #[error("transaction failed")]
42    Transaction(#[from] redb::TransactionError),
43    /// A table could not be opened.
44    #[error("cannot open table")]
45    Table(#[from] redb::TableError),
46    /// A write failed inside the database.
47    #[error("storage write failed")]
48    Storage(#[from] redb::StorageError),
49    /// The commit failed.
50    #[error("commit failed")]
51    Commit(#[from] redb::CommitError),
52    /// Reclaiming the unused pages after the commit failed.
53    #[error("compaction failed")]
54    Compaction(#[from] redb::CompactionError),
55    /// A column's side file could not be written.
56    #[error("cannot write {path}")]
57    Io {
58        /// The file.
59        path: PathBuf,
60        /// The underlying error.
61        #[source]
62        source: std::io::Error,
63    },
64    /// A vocabulary name was registered twice with different ordinals.
65    #[error("{kind} {name:?} is already ordinal {existing}, not {requested}")]
66    Vocabulary {
67        /// The vocabulary.
68        kind: String,
69        /// The name.
70        name: String,
71        /// The ordinal it already has.
72        existing: u32,
73        /// The ordinal requested.
74        requested: u32,
75    },
76}
77
78/// The acceptability that marks a designation as preferred in a language
79/// reference set, as the code system spells it (for SNOMED, the SCTID of
80/// `|Preferred|`).
81#[derive(Debug, Clone, PartialEq, Eq)]
82pub struct PreferredRule {
83    /// The acceptability ordinal meaning preferred.
84    pub preferred: u32,
85    /// The designation use whose preferred designation is a concept display
86    /// (for SNOMED, the synonym), or `None` when the system has no such use.
87    ///
88    /// Only this use goes into the display column. Carrying every use made the
89    /// column a second copy of most of the designations (#322).
90    pub display_use: Option<u32>,
91}
92
93/// An artifact under construction.
94///
95/// Rows are buffered in memory, sorted by key, and written in `finish`, each
96/// table opened once. redb fills pages tightly when keys arrive in order, and
97/// opening a table is not free; the first build wrote every row through its
98/// own `open_table` in arrival order and took ten minutes for the NL edition.
99pub struct StoreBuilder {
100    path: PathBuf,
101    db: Database,
102    system: String,
103    version: String,
104    vocabularies: BTreeMap<&'static str, BTreeMap<u32, String>>,
105    codes: BTreeMap<String, u32>,
106    concepts: BTreeMap<u32, Vec<u8>>,
107    designations: BTreeMap<(u32, u32), Vec<u8>>,
108    acceptability: BTreeMap<(u32, u32, u32), u32>,
109    designation_uses: BTreeMap<(u32, u32), u32>,
110    properties: BTreeMap<(u32, u32), Vec<u8>>,
111}
112
113impl std::fmt::Debug for StoreBuilder {
114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115        f.debug_struct("StoreBuilder")
116            .field("path", &self.path)
117            .field("concepts", &self.concepts.len())
118            .field("designations", &self.designations.len())
119            .finish_non_exhaustive()
120    }
121}
122
123impl StoreBuilder {
124    /// Creates the artifact at `path`, replacing any file there, and records
125    /// the layout version, system, and version.
126    ///
127    /// # Errors
128    ///
129    /// Returns [`BuildError`] when the file cannot be created.
130    pub fn create(path: &Path, system: &str, version: &str) -> Result<Self, BuildError> {
131        if path.exists() {
132            std::fs::remove_file(path).map_err(|source| BuildError::Create {
133                path: path.to_path_buf(),
134                source: redb::DatabaseError::Storage(redb::StorageError::Io(source)),
135            })?;
136        }
137        let db = Database::create(path).map_err(|source| BuildError::Create {
138            path: path.to_path_buf(),
139            source,
140        })?;
141        Ok(Self {
142            path: path.to_path_buf(),
143            db,
144            system: system.to_owned(),
145            version: version.to_owned(),
146            vocabularies: BTreeMap::new(),
147            codes: BTreeMap::new(),
148            concepts: BTreeMap::new(),
149            designations: BTreeMap::new(),
150            acceptability: BTreeMap::new(),
151            designation_uses: BTreeMap::new(),
152            properties: BTreeMap::new(),
153        })
154    }
155
156    /// Names a vocabulary ordinal (a property key, designation use, language
157    /// reference set, or acceptability).
158    ///
159    /// # Errors
160    ///
161    /// Returns [`BuildError::Vocabulary`] when `name` already has another ordinal.
162    pub fn vocabulary(
163        &mut self,
164        kind: crate::store::Vocabulary,
165        ordinal: u32,
166        name: &str,
167    ) -> Result<(), BuildError> {
168        let table_name = match kind {
169            crate::store::Vocabulary::PropertyKeys => tables::PROPERTY_KEYS.name(),
170            crate::store::Vocabulary::DesignationUses => tables::DESIGNATION_USES.name(),
171            crate::store::Vocabulary::LanguageRefsets => tables::LANGUAGE_REFSETS.name(),
172            crate::store::Vocabulary::Acceptabilities => tables::ACCEPTABILITIES.name(),
173        };
174        let entries = self.vocabularies.entry(table_name).or_default();
175        if let Some((existing, _)) = entries
176            .iter()
177            .find(|(key, value)| value.as_str() == name && **key != ordinal)
178        {
179            return Err(BuildError::Vocabulary {
180                kind: table_name.to_owned(),
181                name: name.to_owned(),
182                existing: *existing,
183                requested: ordinal,
184            });
185        }
186        entries.insert(ordinal, name.to_owned());
187        Ok(())
188    }
189
190    /// Adds a concept under `ordinal`.
191    ///
192    /// # Errors
193    ///
194    /// Cannot fail today; the signature keeps the door open for a full buffer.
195    #[expect(
196        clippy::unnecessary_wraps,
197        reason = "the build seam is fallible by contract: a full buffer fails here"
198    )]
199    pub fn concept(&mut self, ordinal: Ordinal, concept: &Concept) -> Result<(), BuildError> {
200        self.codes.insert(concept.code.clone(), ordinal.index());
201        self.concepts.insert(ordinal.index(), concept.encode());
202        Ok(())
203    }
204
205    /// Adds designation `index` of concept `ordinal`.
206    ///
207    /// # Errors
208    ///
209    /// Cannot fail today; the signature keeps the door open for a full buffer.
210    #[expect(
211        clippy::unnecessary_wraps,
212        reason = "the build seam is fallible by contract: a full buffer fails here"
213    )]
214    pub fn designation(
215        &mut self,
216        ordinal: Ordinal,
217        index: u32,
218        designation: &Designation,
219    ) -> Result<(), BuildError> {
220        self.designations
221            .insert((ordinal.index(), index), designation.encode());
222        self.designation_uses
223            .insert((ordinal.index(), index), designation.use_ordinal);
224        Ok(())
225    }
226
227    /// Records the acceptability of a designation in a language reference set.
228    ///
229    /// # Errors
230    ///
231    /// Cannot fail today; the signature keeps the door open for a full buffer.
232    #[expect(
233        clippy::unnecessary_wraps,
234        reason = "the build seam is fallible by contract: a full buffer fails here"
235    )]
236    pub fn acceptability(
237        &mut self,
238        ordinal: Ordinal,
239        index: u32,
240        language_refset: u32,
241        acceptability: u32,
242    ) -> Result<(), BuildError> {
243        self.acceptability
244            .insert((ordinal.index(), index, language_refset), acceptability);
245        Ok(())
246    }
247
248    /// Sets the values of property `key` on concept `ordinal`.
249    ///
250    /// # Errors
251    ///
252    /// Cannot fail today; the signature keeps the door open for a full buffer.
253    #[expect(
254        clippy::unnecessary_wraps,
255        reason = "the build seam is fallible by contract: a full buffer fails here"
256    )]
257    pub fn properties(
258        &mut self,
259        ordinal: Ordinal,
260        key: u32,
261        values: &[PropertyValue],
262    ) -> Result<(), BuildError> {
263        self.properties
264            .insert((ordinal.index(), key), PropertyValue::encode_list(values));
265        Ok(())
266    }
267
268    /// The preferred designation index per `(concept, reference set, use)`:
269    /// the lowest index among those the reference set marks preferred.
270    fn chosen_preferred(&self, rule: &PreferredRule) -> BTreeMap<(u32, u32, u32), u32> {
271        let mut chosen: BTreeMap<(u32, u32, u32), u32> = BTreeMap::new();
272        for ((concept, index, refset), acceptability) in &self.acceptability {
273            if *acceptability != rule.preferred {
274                continue;
275            }
276            let Some(use_ordinal) = self.designation_uses.get(&(*concept, *index)) else {
277                continue;
278            };
279            chosen
280                .entry((*concept, *refset, *use_ordinal))
281                .and_modify(|existing| *existing = (*existing).min(*index))
282                .or_insert(*index);
283        }
284        chosen
285    }
286
287    /// The chosen designations themselves, packed by concept.
288    ///
289    /// Choosing a display is the hottest read the server has, and it should
290    /// not have to find the designation this build already found.
291    fn display_rows(
292        &self,
293        chosen: &BTreeMap<(u32, u32, u32), u32>,
294        display_use: u32,
295    ) -> BTreeMap<u32, Vec<u8>> {
296        let mut by_concept: BTreeMap<u32, Vec<(u32, String, String)>> = BTreeMap::new();
297        for ((concept, refset, use_ordinal), index) in chosen {
298            if *use_ordinal != display_use {
299                continue;
300            }
301            let Some(bytes) = self.designations.get(&(*concept, *index)) else {
302                continue;
303            };
304            let Ok(designation) = Designation::decode(bytes) else {
305                continue;
306            };
307            by_concept.entry(*concept).or_default().push((
308                *refset,
309                designation.language,
310                designation.term,
311            ));
312        }
313        by_concept
314            .into_iter()
315            .map(|(concept, entries)| {
316                let borrowed: Vec<(u32, &str, &str)> = entries
317                    .iter()
318                    .map(|(refset, language, term)| (*refset, language.as_str(), term.as_str()))
319                    .collect();
320                (concept, record::Displays::encode(&borrowed))
321            })
322            .collect()
323    }
324
325    /// Calls `write` once per concept with that concept's rows.
326    ///
327    /// The rows arrive keyed by a concept ordinal followed by the rest of the
328    /// key, so a single pass in key order groups them without sorting again.
329    /// `write` takes the group rather than a finished record so a caller that
330    /// streams to a table holds no second copy of the edition.
331    fn per_concept<'a, K: 'a, V: 'a, R>(
332        rows: &'a BTreeMap<K, V>,
333        concept: impl Fn(&K) -> u32,
334        row: impl Fn(&'a K, &'a V) -> R,
335        mut write: impl FnMut(u32, &[R]) -> Result<(), BuildError>,
336    ) -> Result<(), BuildError> {
337        let mut at = None;
338        let mut group: Vec<R> = Vec::new();
339        for (key, value) in rows {
340            let ordinal = concept(key);
341            if at != Some(ordinal) {
342                if let Some(previous) = at {
343                    write(previous, &group)?;
344                }
345                at = Some(ordinal);
346                group.clear();
347            }
348            group.push(row(key, value));
349        }
350        match at {
351            Some(previous) => write(previous, &group),
352            None => Ok(()),
353        }
354    }
355
356    /// One record per concept, encoded by `pack`, keyed by concept ordinal.
357    fn column_rows<'a, K: 'a, V: 'a, R>(
358        rows: &'a BTreeMap<K, V>,
359        concept: impl Fn(&K) -> u32,
360        row: impl Fn(&'a K, &'a V) -> R,
361        pack: impl Fn(&[R]) -> Vec<u8>,
362    ) -> Result<BTreeMap<u32, Vec<u8>>, BuildError> {
363        let mut packed: BTreeMap<u32, Vec<u8>> = BTreeMap::new();
364        Self::per_concept(rows, concept, row, |ordinal, group| {
365            packed.insert(ordinal, pack(group));
366            Ok(())
367        })?;
368        Ok(packed)
369    }
370
371    /// Writes one dense column to its own file beside the database.
372    ///
373    /// A column is a position-addressed array, which a b-tree of positions
374    /// only pretends to be, and a reader of a file assembles it at the address
375    /// it serves from (#641).
376    fn write_column(
377        &self,
378        name: &str,
379        count: u32,
380        rows: &BTreeMap<u32, Vec<u8>>,
381    ) -> Result<(), BuildError> {
382        let path = Column::file(&self.path, name);
383        let failed = |source| BuildError::Io {
384            path: path.clone(),
385            source,
386        };
387        let file = std::fs::File::create(&path).map_err(failed)?;
388        let mut out = BufWriter::with_capacity(WRITE_BUFFER, file);
389        Column::write_to(
390            count,
391            rows.iter()
392                .map(|(ordinal, bytes)| (Ordinal::new(*ordinal), bytes.as_slice())),
393            &mut out,
394        )
395        .map_err(failed)?;
396        out.flush().map_err(failed)
397    }
398
399    /// Writes every buffered row in key order, computes the preferred
400    /// designations per language reference set and use, records the concept
401    /// count, and commits.
402    ///
403    /// # Errors
404    ///
405    /// Returns [`BuildError`] when a table cannot be written or the commit fails.
406    pub fn finish(self, rule: &PreferredRule) -> Result<PathBuf, BuildError> {
407        let txn = self.db.begin_write()?;
408        {
409            let mut meta = txn.open_table(tables::META)?;
410            meta.insert(tables::META_LAYOUT, tables::LAYOUT_VERSION)?;
411            meta.insert(tables::META_SYSTEM, self.system.as_str())?;
412            meta.insert(tables::META_VERSION, self.version.as_str())?;
413            meta.insert(
414                tables::META_CONCEPTS,
415                self.concepts.len().to_string().as_str(),
416            )?;
417            meta.insert(tables::META_PREFERRED, rule.preferred.to_string().as_str())?;
418        }
419        {
420            let mut codes = txn.open_table(tables::CODES)?;
421            for (code, ordinal) in &self.codes {
422                codes.insert(code.as_str(), *ordinal)?;
423            }
424        }
425        // The count is the highest ordinal, not the entry count, so a gap in
426        // the ordinals cannot push a record off the end of a column.
427        let count = self
428            .concepts
429            .keys()
430            .next_back()
431            .map_or(0, |highest| highest.saturating_add(1));
432        // An ordinal is a position, so these are dense columns in their own
433        // files rather than b-trees keyed by that position.
434        self.write_column(tables::COLUMN_CONCEPTS, count, &self.concepts)?;
435        self.write_column(
436            tables::COLUMN_ACCEPTABILITY,
437            count,
438            &Self::column_rows(
439                &self.acceptability,
440                |(concept, _, _)| *concept,
441                |(_, index, refset), acceptability| (*index, *refset, *acceptability),
442                record::Acceptability::encode,
443            )?,
444        )?;
445        self.write_column(
446            tables::COLUMN_PROPERTIES,
447            count,
448            &Self::column_rows(
449                &self.properties,
450                |(concept, _)| *concept,
451                |(_, key), values| (*key, values.as_slice()),
452                record::Properties::encode,
453            )?,
454        )?;
455        // A code system with no display use still gets the file, empty, so a
456        // store that opens one reads four columns or refuses.
457        let displays = rule.display_use.map_or_else(BTreeMap::new, |display_use| {
458            self.display_rows(&self.chosen_preferred(rule), display_use)
459        });
460        self.write_column(tables::COLUMN_DISPLAYS, count, &displays)?;
461        self.write_designations(&txn)?;
462        for table_def in [
463            tables::PROPERTY_KEYS,
464            tables::DESIGNATION_USES,
465            tables::LANGUAGE_REFSETS,
466            tables::ACCEPTABILITIES,
467        ] {
468            let mut table = txn.open_table(table_def)?;
469            if let Some(entries) = self.vocabularies.get(table_def.name()) {
470                for (ordinal, name) in entries {
471                    table.insert(*ordinal, name.as_str())?;
472                }
473            }
474        }
475        txn.commit()?;
476        // redb grows the file in regions ahead of use; compaction returns the
477        // unused pages so the artifact is the size of its data. Repeated until
478        // redb reports nothing further to reclaim.
479        let mut db = self.db;
480        while db.compact()? {}
481        drop(db);
482        Ok(self.path)
483    }
484
485    /// Writes the designations of each concept as one row.
486    ///
487    /// The designation text is the largest thing the artifact holds, so it
488    /// stays in the database; one row per concept keys the tree by what a
489    /// reader asks for and holds no key per designation (#338).
490    fn write_designations(&self, txn: &redb::WriteTransaction) -> Result<(), BuildError> {
491        let mut designations = txn.open_table(tables::DESIGNATIONS)?;
492        Self::per_concept(
493            &self.designations,
494            |(concept, _)| *concept,
495            |(_, index), bytes| (*index, bytes.as_slice()),
496            |ordinal, group| {
497                designations.insert(ordinal, record::Designations::encode(group).as_slice())?;
498                Ok(())
499            },
500        )
501    }
502}
concept_store/builder.rs

concept_store/
builder.rs
