nodedb_columnar/memtable/
core.rs

1// SPDX-License-Identifier: Apache-2.0
2
3//! `ColumnarMemtable` struct, construction, and plain accessors.
4
5use nodedb_types::columnar::{ColumnDef, ColumnType, ColumnarSchema};
6
7use super::column_data::ColumnData;
8
9/// Default flush threshold: 64K rows per memtable.
10pub const DEFAULT_FLUSH_THRESHOLD: usize = 65_536;
11
12/// In-memory columnar buffer that accumulates INSERTs.
13///
14/// Each column is stored as a typed vector. The memtable flushes to a
15/// compressed segment when the row count reaches the threshold.
16pub struct ColumnarMemtable {
17    pub(super) schema: ColumnarSchema,
18    pub(super) columns: Vec<ColumnData>,
19    pub(super) row_count: usize,
20    pub(super) flush_threshold: usize,
21}
22
23impl ColumnarMemtable {
24    /// Create a new empty memtable for the given schema.
25    pub fn new(schema: &ColumnarSchema) -> Self {
26        Self::with_threshold(schema, DEFAULT_FLUSH_THRESHOLD)
27    }
28
29    /// Create with a custom flush threshold.
30    pub fn with_threshold(schema: &ColumnarSchema, flush_threshold: usize) -> Self {
31        let columns = schema
32            .columns
33            .iter()
34            .map(|col| ColumnData::new(&col.column_type, col.nullable))
35            .collect();
36        Self {
37            schema: schema.clone(),
38            columns,
39            row_count: 0,
40            flush_threshold,
41        }
42    }
43
44    /// Number of rows currently buffered.
45    pub fn row_count(&self) -> usize {
46        self.row_count
47    }
48
49    /// Whether the memtable has reached its flush threshold.
50    pub fn should_flush(&self) -> bool {
51        self.row_count >= self.flush_threshold
52    }
53
54    /// Row count at which [`Self::should_flush`] turns true.
55    pub fn flush_threshold(&self) -> usize {
56        self.flush_threshold
57    }
58
59    /// Whether the memtable is empty.
60    pub fn is_empty(&self) -> bool {
61        self.row_count == 0
62    }
63
64    /// Access the schema.
65    pub fn schema(&self) -> &ColumnarSchema {
66        &self.schema
67    }
68
69    /// Access the raw column data (for the segment writer).
70    pub fn columns(&self) -> &[ColumnData] {
71        &self.columns
72    }
73
74    /// Reconstruct a memtable directly from already-validated column data.
75    ///
76    /// Used by `MutationEngine::from_snapshot` to restore a memtable from a
77    /// backup without re-running per-row validation. The caller must guarantee
78    /// that `columns` is parallel to `schema.columns` and that every column
79    /// contains exactly `row_count` rows. The flush threshold is set to the
80    /// default value.
81    pub(crate) fn from_raw_columns(
82        schema: &ColumnarSchema,
83        columns: Vec<ColumnData>,
84        row_count: usize,
85    ) -> Self {
86        Self {
87            schema: schema.clone(),
88            columns,
89            row_count,
90            flush_threshold: DEFAULT_FLUSH_THRESHOLD,
91        }
92    }
93
94    /// Add a new column to the schema, backfilling existing rows with nulls/defaults.
95    pub fn add_column(&mut self, name: String, column_type: ColumnType, nullable: bool) {
96        if self.schema.columns.iter().any(|c| c.name == name) {
97            return;
98        }
99
100        let existing_rows = self.row_count;
101        let mut col = ColumnData::new(&column_type, nullable);
102        if existing_rows > 0 {
103            col.backfill_nulls(existing_rows);
104        }
105
106        self.columns.push(col);
107        let col_def = if nullable {
108            ColumnDef::nullable(name, column_type)
109        } else {
110            ColumnDef::required(name, column_type)
111        };
112        self.schema.columns.push(col_def);
113    }
114}
nodedb_columnar/memtable/core.rs

nodedb_columnar/memtable/
core.rs
