snomed_ecl_engine/store/
members.rs

1use super::{put_u32, put_u32s, put_u64, sha256, IndexSource, Input};
2use anyhow::{bail, ensure, Result};
3use serde::{Deserialize, Serialize};
4use std::collections::{BTreeMap, HashSet};
5use std::fs::File;
6use std::io::{BufWriter, Write};
7use std::path::Path;
8use std::sync::OnceLock;
9
10/// Rows sorted by referenced component, stored as varint gaps.
11const MAGIC: &[u8; 8] = b"SNMEM003";
12/// Field type of a sorted identifier column written as varint gaps.
13const SORTED_IDS: u32 = 7;
14
15/// The RF2 metadata columns a table keeps, in order, before its own fields.
16///
17/// A member's `id` and `refsetId` are not stored. ECL names reference set
18/// fields from `referencedComponentId` on and gives no meaning to the member
19/// UUID (Appendix E), and every row of a table shares its refset. The UUID was
20/// 16 random bytes a row, a quarter of a packed UK index.
21pub const METADATA: [&str; 4] = [
22    "effectiveTime",
23    "active",
24    "moduleId",
25    "referencedComponentId",
26];
27/// Column of each row's active flag.
28pub const ACTIVE: usize = 1;
29/// Column of each row's referenced component.
30pub const REFERENCES: usize = 3;
31/// Where a table's own fields begin.
32pub const FIELDS: usize = 4;
33
34#[derive(Clone, Debug, Serialize, Deserialize)]
35pub struct MemberManifest {
36    pub refset: u64,
37    pub bytes: u64,
38    pub sha256: String,
39    pub rows: usize,
40    pub fields: Vec<String>,
41}
42
43#[derive(Clone, Debug)]
44pub struct TextColumn {
45    pub offsets: Vec<u32>,
46    pub text: String,
47}
48impl Default for TextColumn {
49    fn default() -> Self {
50        Self {
51            offsets: vec![0],
52            text: String::new(),
53        }
54    }
55}
56impl TextColumn {
57    pub fn push(&mut self, value: &str) -> Result<()> {
58        if self.offsets.is_empty() {
59            self.offsets.push(0);
60        }
61        self.text.push_str(value);
62        self.offsets.push(u32::try_from(self.text.len())?);
63        Ok(())
64    }
65    pub fn get(&self, row: usize) -> &str {
66        &self.text[self.offsets[row] as usize..self.offsets[row + 1] as usize]
67    }
68}
69
70#[derive(Clone, Debug)]
71pub enum MemberColumn {
72    Id(Vec<u64>),
73    Integer(Vec<i64>),
74    Number(TextColumn),
75    Boolean(Vec<u8>),
76    Time(Vec<u32>),
77    Text(TextColumn),
78    Uuid(Vec<[u8; 16]>),
79}
80
81#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize)]
82#[serde(tag = "type", content = "value", rename_all = "snake_case")]
83pub enum MemberValue {
84    Concept(String),
85    /// A description, relationship or other non-concept SCTID held by a component field.
86    Component(String),
87    Number(String),
88    Boolean(bool),
89    Time(String),
90    String(String),
91}
92
93/// RF2 partition digits 00 and 10 identify concepts.
94pub fn is_concept_id(id: u64) -> bool {
95    matches!((id / 10) % 100, 0 | 10)
96}
97
98impl MemberColumn {
99    pub fn len(&self) -> usize {
100        match self {
101            Self::Id(v) => v.len(),
102            Self::Integer(v) => v.len(),
103            Self::Boolean(v) => v.len(),
104            Self::Time(v) => v.len(),
105            Self::Text(v) | Self::Number(v) => v.offsets.len().saturating_sub(1),
106            Self::Uuid(v) => v.len(),
107        }
108    }
109    pub fn is_empty(&self) -> bool {
110        self.len() == 0
111    }
112    /// The rows at `order`, in that order.
113    fn select(&self, order: &[usize]) -> Result<Self> {
114        fn pick<T: Copy>(values: &[T], order: &[usize]) -> Vec<T> {
115            order.iter().map(|&row| values[row]).collect()
116        }
117        let text = |v: &TextColumn| -> Result<TextColumn> {
118            let mut out = TextColumn::default();
119            for &row in order {
120                out.push(v.get(row))?;
121            }
122            Ok(out)
123        };
124        Ok(match self {
125            Self::Id(v) => Self::Id(pick(v, order)),
126            Self::Integer(v) => Self::Integer(pick(v, order)),
127            Self::Number(v) => Self::Number(text(v)?),
128            Self::Boolean(v) => Self::Boolean(pick(v, order)),
129            Self::Time(v) => Self::Time(pick(v, order)),
130            Self::Text(v) => Self::Text(text(v)?),
131            Self::Uuid(v) => Self::Uuid(pick(v, order)),
132        })
133    }
134    pub fn value(&self, row: usize) -> MemberValue {
135        match self {
136            Self::Id(v) if is_concept_id(v[row]) => MemberValue::Concept(v[row].to_string()),
137            Self::Id(v) => MemberValue::Component(v[row].to_string()),
138            Self::Integer(v) => MemberValue::Number(v[row].to_string()),
139            Self::Number(v) => MemberValue::Number(v.get(row).into()),
140            Self::Boolean(v) => MemberValue::Boolean(v[row] != 0),
141            Self::Time(v) => MemberValue::Time(if v[row] == 0 {
142                String::new()
143            } else {
144                format!("{:08}", v[row])
145            }),
146            Self::Text(v) => MemberValue::String(v.get(row).into()),
147            Self::Uuid(v) => MemberValue::String(format_uuid(&v[row])),
148        }
149    }
150}
151
152pub fn parse_uuid(text: &str) -> Result<[u8; 16]> {
153    ensure!(
154        text.len() == 36 && [8, 13, 18, 23].iter().all(|&i| text.as_bytes()[i] == b'-'),
155        "Invalid member UUID"
156    );
157    let hex: String = text.chars().filter(|&c| c != '-').collect();
158    ensure!(
159        hex.len() == 32 && hex.bytes().all(|b| b.is_ascii_hexdigit()),
160        "Invalid member UUID"
161    );
162    let mut uuid = [0; 16];
163    for (i, byte) in uuid.iter_mut().enumerate() {
164        *byte = u8::from_str_radix(&hex[i * 2..i * 2 + 2], 16)?;
165    }
166    Ok(uuid)
167}
168pub fn format_uuid(uuid: &[u8; 16]) -> String {
169    let mut text = String::with_capacity(36);
170    for (i, b) in uuid.iter().enumerate() {
171        if [4, 6, 8, 10].contains(&i) {
172            text.push('-');
173        }
174        use std::fmt::Write;
175        write!(text, "{b:02x}").unwrap();
176    }
177    text
178}
179
180/// One concept-based refset. Field columns preserve member rows and their metadata.
181#[derive(Clone, Debug)]
182pub struct MemberTable {
183    pub refset: u64,
184    pub names: Vec<String>,
185    pub columns: Vec<MemberColumn>,
186}
187impl MemberTable {
188    pub fn append(&mut self, other: Self) -> Result<()> {
189        self.validate()?;
190        other.validate()?;
191        ensure!(
192            self.refset == other.refset && self.names == other.names,
193            "Refset schemas differ"
194        );
195        for (left, right) in self.columns.iter_mut().zip(other.columns) {
196            // An integer column promoted to decimal text on either side merges exactly.
197            if let (MemberColumn::Integer(a), MemberColumn::Number(_)) = (&*left, &right) {
198                let mut text = TextColumn::default();
199                for value in a {
200                    text.push(&value.to_string())?;
201                }
202                *left = MemberColumn::Number(text);
203            }
204            let right = match (&*left, right) {
205                (MemberColumn::Number(_), MemberColumn::Integer(b)) => {
206                    let mut text = TextColumn::default();
207                    for value in b {
208                        text.push(&value.to_string())?;
209                    }
210                    MemberColumn::Number(text)
211                }
212                (_, right) => right,
213            };
214            match (left, right) {
215                (MemberColumn::Id(a), MemberColumn::Id(b)) => a.extend(b),
216                (MemberColumn::Integer(a), MemberColumn::Integer(b)) => a.extend(b),
217                (MemberColumn::Boolean(a), MemberColumn::Boolean(b)) => a.extend(b),
218                (MemberColumn::Time(a), MemberColumn::Time(b)) => a.extend(b),
219                (MemberColumn::Uuid(a), MemberColumn::Uuid(b)) => a.extend(b),
220                (MemberColumn::Text(a), MemberColumn::Text(b))
221                | (MemberColumn::Number(a), MemberColumn::Number(b)) => {
222                    let start = u32::try_from(a.text.len())?;
223                    for offset in b.offsets.into_iter().skip(1) {
224                        a.offsets.push(
225                            start.checked_add(offset).ok_or_else(|| {
226                                anyhow::anyhow!("Member text exceeds u32 capacity")
227                            })?,
228                        );
229                    }
230                    a.text.push_str(&b.text);
231                }
232                _ => bail!("Refset field types differ"),
233            }
234        }
235        self.validate()
236    }
237    pub fn len(&self) -> usize {
238        self.columns.first().map_or(0, MemberColumn::len)
239    }
240    pub fn is_empty(&self) -> bool {
241        self.len() == 0
242    }
243    pub fn column(&self, name: &str) -> Option<&MemberColumn> {
244        self.names
245            .iter()
246            .position(|n| n.eq_ignore_ascii_case(name))
247            .map(|i| &self.columns[i])
248    }
249    pub fn validate(&self) -> Result<()> {
250        ensure!(
251            self.names.len() == self.columns.len()
252                && self.names.len() >= FIELDS
253                && self.names.len() <= 64,
254            "Invalid member schema"
255        );
256        ensure!(
257            self.names[..FIELDS] == METADATA,
258            "Invalid member metadata fields"
259        );
260        let mut names = HashSet::new();
261        for (name, col) in self.names.iter().zip(&self.columns) {
262            ensure!(
263                !name.is_empty()
264                    && name.bytes().all(|b| b.is_ascii_alphabetic())
265                    && names.insert(name.to_ascii_lowercase()),
266                "Invalid or duplicate member field"
267            );
268            ensure!(col.len() == self.len(), "Member column lengths differ");
269            match col {
270                MemberColumn::Id(v) => ensure!(
271                    v.iter()
272                        .all(|id| *id > 0 && *id < 1_000_000_000_000_000_000),
273                    "Invalid member SCTID"
274                ),
275                MemberColumn::Boolean(v) => {
276                    ensure!(v.iter().all(|&v| v <= 1), "Invalid member Boolean")
277                }
278                MemberColumn::Time(v) => {
279                    ensure!(v.iter().all(|&v| valid_time(v)), "Invalid member date")
280                }
281                MemberColumn::Text(v) | MemberColumn::Number(v) => {
282                    ensure!(
283                        v.offsets.first() == Some(&0)
284                            && v.offsets.last().copied() == u32::try_from(v.text.len()).ok(),
285                        "Invalid member text offsets"
286                    );
287                    ensure!(
288                        v.offsets.windows(2).all(|w| w[0] <= w[1])
289                            && v.offsets
290                                .iter()
291                                .all(|&i| v.text.is_char_boundary(i as usize)),
292                        "Invalid member UTF-8 boundary"
293                    );
294                    if matches!(col, MemberColumn::Number(_)) {
295                        ensure!(
296                            (0..v.offsets.len() - 1)
297                                .all(|i| crate::decimal::Decimal::parse(v.get(i)).is_some()),
298                            "Invalid numeric member field"
299                        );
300                    }
301                }
302                _ => {}
303            }
304        }
305        ensure!(
306            matches!(&self.columns[0], MemberColumn::Time(_))
307                && matches!(&self.columns[ACTIVE], MemberColumn::Boolean(_)),
308            "Invalid member status metadata"
309        );
310        ensure!(
311            matches!(&self.columns[2], MemberColumn::Id(_)),
312            "Invalid member module metadata"
313        );
314        ensure!(
315            matches!(&self.columns[REFERENCES], MemberColumn::Id(v) if v.iter().all(|id| matches!((id / 10) % 100, 0 | 10))),
316            "Refset does not reference concepts"
317        );
318        Ok(())
319    }
320    /// Writes rows sorted by referenced component, ties in their current order.
321    ///
322    /// Sorted, the referenced components are small gaps from one row to the
323    /// next, and neighbouring rows of a map share most of their other fields
324    /// too: the UK tables pack to under half their size in release order.
325    pub fn write(&self, directory: &Path) -> Result<MemberManifest> {
326        self.validate()?;
327        let MemberColumn::Id(references) = &self.columns[REFERENCES] else {
328            bail!("Invalid member reference column");
329        };
330        let mut order: Vec<usize> = (0..self.len()).collect();
331        order.sort_by_key(|&row| references[row]);
332        std::fs::create_dir_all(directory.join("members"))?;
333        let path = directory
334            .join("members")
335            .join(format!("{}.bin", self.refset));
336        let mut out = BufWriter::new(File::create_new(&path)?);
337        out.write_all(MAGIC)?;
338        put_u64(&mut out, self.refset)?;
339        let schema = serde_json::to_vec(&self.names)?;
340        put_u64(&mut out, schema.len() as u64)?;
341        out.write_all(&schema)?;
342        for (index, column) in self.columns.iter().enumerate() {
343            let column = &column.select(&order)?;
344            match column {
345                MemberColumn::Id(v) if index == REFERENCES => {
346                    let bytes = super::varint::encode_u64(v)?;
347                    put_u32(&mut out, SORTED_IDS)?;
348                    put_u64(&mut out, v.len() as u64)?;
349                    put_u64(&mut out, bytes.len() as u64)?;
350                    out.write_all(&bytes)?;
351                }
352                MemberColumn::Id(v) => {
353                    put_u32(&mut out, 0)?;
354                    put_u64(&mut out, v.len() as u64)?;
355                    for &n in v {
356                        put_u64(&mut out, n)?;
357                    }
358                }
359                MemberColumn::Integer(v) => {
360                    put_u32(&mut out, 1)?;
361                    put_u64(&mut out, v.len() as u64)?;
362                    for &n in v {
363                        put_u64(&mut out, n as u64)?;
364                    }
365                }
366                MemberColumn::Boolean(v) => {
367                    put_u32(&mut out, 2)?;
368                    put_u64(&mut out, v.len() as u64)?;
369                    out.write_all(v)?;
370                }
371                MemberColumn::Time(v) => {
372                    put_u32(&mut out, 3)?;
373                    put_u32s(&mut out, v)?;
374                }
375                MemberColumn::Text(v) | MemberColumn::Number(v) => {
376                    put_u32(
377                        &mut out,
378                        if matches!(column, MemberColumn::Number(_)) {
379                            6
380                        } else {
381                            4
382                        },
383                    )?;
384                    put_u32s(&mut out, &v.offsets)?;
385                    put_u64(&mut out, v.text.len() as u64)?;
386                    out.write_all(v.text.as_bytes())?;
387                }
388                MemberColumn::Uuid(v) => {
389                    put_u32(&mut out, 5)?;
390                    put_u64(&mut out, v.len() as u64)?;
391                    for uuid in v {
392                        out.write_all(uuid)?;
393                    }
394                }
395            }
396        }
397        out.flush()?;
398        out.get_ref().sync_all()?;
399        Ok(MemberManifest {
400            refset: self.refset,
401            bytes: path.metadata()?.len(),
402            sha256: sha256(&path)?,
403            rows: self.len(),
404            fields: self.names.clone(),
405        })
406    }
407    pub(super) fn open(source: &IndexSource, metadata: &MemberManifest) -> Result<Self> {
408        let mut input = Input::open(
409            &source.section(&format!("members/{}.bin", metadata.refset))?,
410            MAGIC,
411        )?;
412        let refset = input.u64()?;
413        let names: Vec<String> = serde_json::from_slice(&input.bytes()?)?;
414        ensure!(
415            names.len() <= 64 && names == metadata.fields,
416            "Member schema differs from manifest"
417        );
418        let mut columns = Vec::new();
419        for _ in &names {
420            columns.push(match input.u32()? {
421                0 => MemberColumn::Id({
422                    let n = input.count(8)?;
423                    (0..n).map(|_| input.u64()).collect::<Result<_>>()?
424                }),
425                1 => MemberColumn::Integer({
426                    let n = input.count(8)?;
427                    (0..n)
428                        .map(|_| input.u64().map(|v| v as i64))
429                        .collect::<Result<_>>()?
430                }),
431                2 => MemberColumn::Boolean(input.bytes()?),
432                3 => MemberColumn::Time(input.u32s()?),
433                4 => MemberColumn::Text(TextColumn {
434                    offsets: input.u32s()?,
435                    text: String::from_utf8(input.bytes()?)?,
436                }),
437                5 => {
438                    let n = input.count(16)?;
439                    let mut v = Vec::with_capacity(n);
440                    for _ in 0..n {
441                        let mut uuid = [0; 16];
442                        input.read(&mut uuid)?;
443                        v.push(uuid);
444                    }
445                    MemberColumn::Uuid(v)
446                }
447                6 => MemberColumn::Number(TextColumn {
448                    offsets: input.u32s()?,
449                    text: String::from_utf8(input.bytes()?)?,
450                }),
451                SORTED_IDS => {
452                    let n = input.count(1)?;
453                    MemberColumn::Id(super::varint::decode_u64(n, &input.bytes()?)?)
454                }
455                _ => bail!("Unknown member field type"),
456            });
457        }
458        ensure!(
459            input.remaining == 0 && refset == metadata.refset,
460            "Member manifest counts differ or trailing bytes"
461        );
462        let table = Self {
463            refset,
464            names,
465            columns,
466        };
467        ensure!(
468            table.len() == metadata.rows,
469            "Member manifest counts differ"
470        );
471        table.validate()?;
472        Ok(table)
473    }
474}
475
476pub(crate) fn valid_time(value: u32) -> bool {
477    if value == 0 {
478        return true;
479    }
480    let (year, month, day) = (value / 10000, value / 100 % 100, value % 100);
481    let days = match month {
482        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
483        4 | 6 | 9 | 11 => 30,
484        2 => 28 + u32::from(year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)),
485        _ => 0,
486    };
487    (1000..=9999).contains(&year) && day >= 1 && day <= days
488}
489
490/// A table, opened on first use, and per-column row orders built on first use.
491#[derive(Debug)]
492struct Slot {
493    meta: MemberManifest,
494    /// Field names in the current layout, whichever layout the manifest records.
495    fields: Vec<String>,
496    table: OnceLock<std::result::Result<MemberTable, String>>,
497    orders: Box<[OnceLock<Vec<u32>>]>,
498}
499impl Slot {
500    fn new(
501        meta: MemberManifest,
502        table: OnceLock<std::result::Result<MemberTable, String>>,
503    ) -> Self {
504        let fields = meta.fields.clone();
505        let orders = (0..fields.len()).map(|_| OnceLock::new()).collect();
506        Self {
507            meta,
508            fields,
509            table,
510            orders,
511        }
512    }
513}
514
515#[derive(Debug, Default)]
516pub struct MemberStore {
517    source: Option<IndexSource>,
518    tables: BTreeMap<u64, Slot>,
519    available: bool,
520}
521impl MemberStore {
522    pub fn loaded(tables: Vec<MemberTable>) -> Result<Self> {
523        let mut result = Self {
524            available: true,
525            ..Self::default()
526        };
527        for table in tables {
528            table.validate()?;
529            let meta = MemberManifest {
530                refset: table.refset,
531                bytes: 0,
532                sha256: String::new(),
533                rows: table.len(),
534                fields: table.names.clone(),
535            };
536            ensure!(
537                result
538                    .tables
539                    .insert(table.refset, Slot::new(meta, OnceLock::from(Ok(table))))
540                    .is_none(),
541                "Duplicate member table"
542            );
543        }
544        Ok(result)
545    }
546    pub(super) fn lazy(source: &IndexSource, metadata: Vec<MemberManifest>) -> Result<Self> {
547        let mut store = Self {
548            source: Some(source.clone()),
549            available: true,
550            ..Self::default()
551        };
552        for meta in metadata {
553            ensure!(
554                meta.refset > 0
555                    && store
556                        .tables
557                        .insert(meta.refset, Slot::new(meta, OnceLock::new()))
558                        .is_none(),
559                "Duplicate member manifest"
560            );
561        }
562        Ok(store)
563    }
564    pub fn is_available(&self) -> bool {
565        self.available
566    }
567    pub fn refsets(&self) -> impl Iterator<Item = u64> + '_ {
568        self.tables.keys().copied()
569    }
570    pub fn fields(&self, refset: u64) -> Option<&[String]> {
571        self.tables.get(&refset).map(|slot| slot.fields.as_slice())
572    }
573    pub fn get(&self, refset: u64) -> Result<Option<&MemberTable>> {
574        let Some(slot) = self.tables.get(&refset) else {
575            return Ok(None);
576        };
577        match slot.table.get_or_init(|| {
578            MemberTable::open(self.source.as_ref().unwrap(), &slot.meta).map_err(|e| e.to_string())
579        }) {
580            Ok(table) => Ok(Some(table)),
581            Err(e) => bail!("Member index: {e}"),
582        }
583    }
584    /// Rows of an identifier column ordered by value, then row; `None` for
585    /// other column types. Built on first use and kept with the table.
586    pub fn order(&self, refset: u64, column: usize) -> Result<Option<&[u32]>> {
587        let Some(MemberColumn::Id(values)) = self.get(refset)?.and_then(|t| t.columns.get(column))
588        else {
589            return Ok(None);
590        };
591        let order = self.tables[&refset].orders[column].get_or_init(|| {
592            let mut rows: Vec<u32> = (0..values.len() as u32).collect();
593            rows.sort_by_key(|&row| values[row as usize]);
594            rows
595        });
596        Ok(Some(order))
597    }
598}
snomed_ecl_engine/store/members.rs

snomed_ecl_engine/store/
members.rs
