Skip to main content

day_persistence/
blob.rs

1// Copyright © The Daybrite Project
2// SPDX-License-Identifier: MPL-2.0
3
4//! Lazy blob columns: a [`Blob`] field is a `BLOB` column that a row fault never reads. The
5//! row arrives with the column as `NULL` (the fault `SELECT` names it as a literal `NULL`),
6//! the field decodes to [`Blob::stored`], and the bytes come when asked for, one point
7//! `SELECT` per read, through a memo bounded in bytes rather than rows
8//! ([`ModelContainer::set_blob_cache_limit`]).
9//!
10//! Writes travel inside the struct: [`Blob::new`] holds the bytes as *pending* until the
11//! flush writes them, after which the container settles the field back to `stored` under
12//! its own author, so nothing re-dirties, and parks the bytes in the memo so the next read
13//! is a hit. A pending value answers reads directly, so a write made this turn is visible
14//! before it is flushed, like any other field.
15//!
16//! The upsert the fold emits protects a stored value: a lazy column's conflict clause is
17//! `c = COALESCE(excluded.c, c)`, and an `UPDATE` carrying a `NULL` for a lazy column (a
18//! stored value the row rewrote with no new bytes) skips that column, so rewriting a row's
19//! other fields never erases the bytes the row already has.
20//!
21//! The derive places lazy columns last in a new table's DDL. SQLite stores a large value in
22//! an overflow chain and a select that stops at an earlier column never reads that chain,
23//! so even raw `SELECT id, filename` over the table stays cheap.
24
25use std::collections::{HashMap, VecDeque};
26use std::ops::Range;
27use std::sync::Arc;
28
29use crate::{ColumnValue, DbError, Model, ModelContainer, SqlType, Value};
30
31/// A lazy `BLOB` column. `Default` is the stored, unread state; [`Blob::new`] carries bytes
32/// to the next flush. The bytes are read through the field's accessor
33/// ([`BlobField::bytes`] and friends), never through the value itself.
34#[derive(Clone, Default)]
35pub struct Blob {
36    pending: Option<Arc<Vec<u8>>>,
37}
38
39impl Blob {
40    /// Bytes to write at the next flush.
41    pub fn new(bytes: impl Into<Vec<u8>>) -> Blob {
42        Blob {
43            pending: Some(Arc::new(bytes.into())),
44        }
45    }
46
47    /// The stored state: whatever the file holds, read on demand.
48    pub const fn stored() -> Blob {
49        Blob { pending: None }
50    }
51
52    /// Whether this value carries bytes the flush has not written yet.
53    pub fn is_pending(&self) -> bool {
54        self.pending.is_some()
55    }
56
57    /// The unflushed bytes, if any.
58    pub fn pending(&self) -> Option<&Arc<Vec<u8>>> {
59        self.pending.as_ref()
60    }
61
62    /// Hand the pending bytes over and become stored: what the container does after the
63    /// flush that wrote them.
64    pub fn take_pending(&mut self) -> Option<Arc<Vec<u8>>> {
65        self.pending.take()
66    }
67}
68
69impl PartialEq for Blob {
70    /// Two stored values are equal (both mean "the file's bytes"); pending values compare by
71    /// content; a pending and a stored value differ, since one has an unflushed write.
72    fn eq(&self, other: &Self) -> bool {
73        match (&self.pending, &other.pending) {
74            (None, None) => true,
75            (Some(a), Some(b)) => a == b,
76            _ => false,
77        }
78    }
79}
80
81impl std::fmt::Debug for Blob {
82    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
83        match &self.pending {
84            Some(b) => write!(f, "Blob(pending {} bytes)", b.len()),
85            None => f.write_str("Blob(stored)"),
86        }
87    }
88}
89
90impl ColumnValue for Blob {
91    const SQL_TYPE: SqlType = SqlType::Blob;
92    /// Pending bytes go to the file; a stored value writes `NULL`, which the fold's upsert
93    /// and update statements treat as "leave the column alone".
94    fn to_sqlite_value(&self) -> Value {
95        match &self.pending {
96            Some(b) => Value::Blob(b.as_ref().clone()),
97            None => Value::Null,
98        }
99    }
100    /// A fault never reads the column, so whatever arrives decodes to the stored state.
101    fn from_sqlite_value(_: Value) -> Result<Self, DbError> {
102        Ok(Blob::stored())
103    }
104}
105
106// ---------------------------------------------------------------------------------------------
107// The accessor surface
108// ---------------------------------------------------------------------------------------------
109
110/// What a [`Blob`] field's accessor offers beyond the plain field: the bytes, on demand.
111///
112/// Every read is tracked through the field, so a view that shows the bytes re-renders when
113/// they are written. A pending value (written this turn, not yet flushed) answers from
114/// memory; otherwise the container answers from its byte-bounded memo or the file.
115pub trait BlobField {
116    /// The whole value. Empty when the column is `NULL` or the row is gone.
117    fn bytes(&self) -> Result<Arc<Vec<u8>>, DbError>;
118    /// The value's length in bytes, without reading it (`length(col)` in SQL).
119    fn len(&self) -> Result<usize, DbError>;
120    /// Whether the value is empty or `NULL`.
121    fn is_empty(&self) -> Result<bool, DbError> {
122        Ok(self.len()? == 0)
123    }
124    /// `range` of the value, clamped to its length: `substr(col, …)` in SQL, so only the
125    /// requested bytes cross from the engine.
126    fn read_range(&self, range: Range<usize>) -> Result<Vec<u8>, DbError>;
127    /// Write new bytes: the field becomes pending until the next flush.
128    fn write_bytes(&self, bytes: impl Into<Vec<u8>>);
129    /// A `std::io::Read` over the value, reading it in chunks through [`BlobField::read_range`].
130    fn reader(&self) -> BlobReader;
131}
132
133impl<S, M> BlobField for day_model::Field<S, M, Blob>
134where
135    S: day_model::Source<M> + Copy + 'static,
136    M: Model + 'static,
137{
138    fn bytes(&self) -> Result<Arc<Vec<u8>>, DbError> {
139        if let Some(pending) = day_reactive::Binding::read(self).pending {
140            return Ok(pending);
141        }
142        let (container, store, key) = locate(*self)?;
143        container.blob_bytes(store, key, column_of::<M>(self.label()))
144    }
145
146    fn len(&self) -> Result<usize, DbError> {
147        if let Some(pending) = day_reactive::Binding::read(self).pending {
148            return Ok(pending.len());
149        }
150        let (container, store, key) = locate(*self)?;
151        container.blob_len(store, key, column_of::<M>(self.label()))
152    }
153
154    fn read_range(&self, range: Range<usize>) -> Result<Vec<u8>, DbError> {
155        if let Some(pending) = day_reactive::Binding::read(self).pending {
156            return Ok(slice(&pending, range));
157        }
158        let (container, store, key) = locate(*self)?;
159        container.blob_range(store, key, column_of::<M>(self.label()), range)
160    }
161
162    fn write_bytes(&self, bytes: impl Into<Vec<u8>>) {
163        // The memo would otherwise keep answering with the old bytes until the flush.
164        if let Ok((container, store, key)) = locate(*self) {
165            container.blob_forget_field(store, key, column_of::<M>(self.label()));
166        }
167        day_reactive::Binding::write(self, Blob::new(bytes));
168    }
169
170    fn reader(&self) -> BlobReader {
171        let field = *self;
172        BlobReader {
173            read_range: Box::new(move |r| field.read_range(r)),
174            len: Box::new(move || field.len()),
175            pos: 0,
176            end: None,
177        }
178    }
179}
180
181type RangeReadFn = Box<dyn Fn(Range<usize>) -> Result<Vec<u8>, DbError>>;
182type LenFn = Box<dyn Fn() -> Result<usize, DbError>>;
183
184/// A `std::io::Read` over a lazy blob, in 256 KiB chunks.
185pub struct BlobReader {
186    read_range: RangeReadFn,
187    len: LenFn,
188    pos: usize,
189    end: Option<usize>,
190}
191
192impl BlobReader {
193    /// How much of the value one `read` asks the engine for at most.
194    pub const CHUNK: usize = 256 * 1024;
195}
196
197impl std::io::Read for BlobReader {
198    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
199        let end = match self.end {
200            Some(e) => e,
201            None => {
202                let e = (self.len)().map_err(std::io::Error::other)?;
203                self.end = Some(e);
204                e
205            }
206        };
207        if self.pos >= end || buf.is_empty() {
208            return Ok(0);
209        }
210        let want = buf.len().min(Self::CHUNK).min(end - self.pos);
211        let chunk = (self.read_range)(self.pos..self.pos + want).map_err(std::io::Error::other)?;
212        let n = chunk.len().min(buf.len());
213        buf[..n].copy_from_slice(&chunk[..n]);
214        self.pos += n;
215        Ok(n)
216    }
217}
218
219fn slice(bytes: &[u8], range: Range<usize>) -> Vec<u8> {
220    let start = range.start.min(bytes.len());
221    let end = range.end.clamp(start, bytes.len());
222    bytes[start..end].to_vec()
223}
224
225/// The stored column name of the field labelled `field` on `M`.
226fn column_of<M: Model>(field: &'static str) -> &'static str {
227    M::COLUMNS
228        .iter()
229        .find(|c| c.field == field)
230        .map(|c| c.name)
231        .unwrap_or(field)
232}
233
234/// The container holding this field's store, with the store id and the row's key handle.
235fn locate<S, M>(field: day_model::Field<S, M, Blob>) -> Result<(ModelContainer, u64, u64), DbError>
236where
237    S: day_model::Source<M> + Copy + 'static,
238    M: 'static,
239{
240    let mut parts = Vec::new();
241    day_model::Source::components(field, &mut parts);
242    let (Some(&store), Some(&key)) = (parts.first(), parts.get(1)) else {
243        return Err(DbError::driver(
244            "a Blob field can only be read through a row of a container's store",
245        ));
246    };
247    crate::relations::container_for_store(store)
248        .map(|c| (c, store, key))
249        .ok_or_else(|| {
250            DbError::driver("a Blob field was read on a store no open ModelContainer holds")
251        })
252}
253
254// ---------------------------------------------------------------------------------------------
255// The memo
256// ---------------------------------------------------------------------------------------------
257
258/// Bytes read or written recently, bounded in bytes: the oldest entry leaves first, and a
259/// value larger than the whole limit is served but never kept.
260#[derive(Default)]
261pub(crate) struct BlobMemo {
262    entries: HashMap<(u64, u64, &'static str), Arc<Vec<u8>>>,
263    order: VecDeque<(u64, u64, &'static str)>,
264    bytes: usize,
265}
266
267impl BlobMemo {
268    pub(crate) fn get(&mut self, k: (u64, u64, &'static str)) -> Option<Arc<Vec<u8>>> {
269        let hit = self.entries.get(&k).cloned()?;
270        // Most recently used goes to the back.
271        if let Some(i) = self.order.iter().position(|e| *e == k) {
272            self.order.remove(i);
273            self.order.push_back(k);
274        }
275        Some(hit)
276    }
277
278    pub(crate) fn insert(&mut self, k: (u64, u64, &'static str), v: Arc<Vec<u8>>, limit: usize) {
279        self.forget_one(k);
280        if v.len() > limit {
281            return;
282        }
283        self.bytes += v.len();
284        self.entries.insert(k, v);
285        self.order.push_back(k);
286        while self.bytes > limit {
287            let Some(old) = self.order.pop_front() else {
288                break;
289            };
290            if let Some(v) = self.entries.remove(&old) {
291                self.bytes -= v.len();
292            }
293        }
294    }
295
296    pub(crate) fn forget_one(&mut self, k: (u64, u64, &'static str)) {
297        if let Some(v) = self.entries.remove(&k) {
298            self.bytes -= v.len();
299            if let Some(i) = self.order.iter().position(|e| *e == k) {
300                self.order.remove(i);
301            }
302        }
303    }
304
305    /// Drop every entry of one row (any column).
306    pub(crate) fn forget_row(&mut self, store: u64, key: u64) {
307        let keys: Vec<_> = self
308            .entries
309            .keys()
310            .filter(|(s, k, _)| *s == store && *k == key)
311            .copied()
312            .collect();
313        for k in keys {
314            self.forget_one(k);
315        }
316    }
317
318    /// Drop every entry of one store.
319    pub(crate) fn forget_store(&mut self, store: u64) {
320        let keys: Vec<_> = self
321            .entries
322            .keys()
323            .filter(|(s, _, _)| *s == store)
324            .copied()
325            .collect();
326        for k in keys {
327            self.forget_one(k);
328        }
329    }
330
331    pub(crate) fn clear(&mut self) {
332        self.entries.clear();
333        self.order.clear();
334        self.bytes = 0;
335    }
336
337    pub(crate) fn bytes(&self) -> usize {
338        self.bytes
339    }
340
341    pub(crate) fn len(&self) -> usize {
342        self.entries.len()
343    }
344}