Skip to main content

day_persistence/
blob.rs

1// Copyright © The Daybrite Project
2// SPDX-License-Identifier: MPL-2.0
3
4//! Lazy blob columns: a [`Blob`] field is a `BLOB` column that a row fault never reads. The
5//! row arrives with the column as `NULL` (the fault `SELECT` names it as a literal `NULL`),
6//! the field decodes to [`Blob::stored`], and the bytes come when asked for, one point
7//! `SELECT` per read, through a memo bounded in bytes rather than rows
8//! ([`ModelContainer::set_blob_cache_limit`]).
9//!
10//! Writes travel inside the struct: [`Blob::new`] holds the bytes as *pending* until the
11//! flush writes them, after which the container settles the field back to `stored` under
12//! its own author, so nothing re-dirties, and parks the bytes in the memo so the next read
13//! is a hit. A pending value answers reads directly, so a write made this turn is visible
14//! before it is flushed, like any other field.
15//!
16//! The upsert the fold emits protects a stored value: a lazy column's conflict clause is
17//! `c = COALESCE(excluded.c, c)`, and an `UPDATE` carrying a `NULL` for a lazy column (a
18//! stored value the row rewrote with no new bytes) skips that column, so rewriting a row's
19//! other fields never erases the bytes the row already has.
20//!
21//! The derive places lazy columns last in a new table's DDL. SQLite stores a large value in
22//! an overflow chain and a select that stops at an earlier column never reads that chain,
23//! so even raw `SELECT id, filename` over the table stays cheap.
24
25use std::collections::{HashMap, VecDeque};
26use std::ops::Range;
27use std::sync::Arc;
28
29use crate::{ColumnValue, DbError, Model, ModelContainer, SqlType, Value};
30
31/// A lazy `BLOB` column. `Default` is the stored, unread state; [`Blob::new`] carries bytes
32/// to the next flush. The bytes are read through the field's accessor
33/// ([`BlobField::bytes`] and friends), never through the value itself.
34#[derive(Clone, Default)]
35pub struct Blob {
36    pending: Option<Arc<Vec<u8>>>,
37}
38
39impl Blob {
40    /// Bytes to write at the next flush.
41    pub fn new(bytes: impl Into<Vec<u8>>) -> Blob {
42        Blob {
43            pending: Some(Arc::new(bytes.into())),
44        }
45    }
46
47    /// The stored state: whatever the file holds, read on demand.
48    pub const fn stored() -> Blob {
49        Blob { pending: None }
50    }
51
52    /// Whether this value carries bytes the flush has not written yet.
53    pub fn is_pending(&self) -> bool {
54        self.pending.is_some()
55    }
56
57    /// The unflushed bytes, if any.
58    pub fn pending(&self) -> Option<&Arc<Vec<u8>>> {
59        self.pending.as_ref()
60    }
61
62    /// Hand the pending bytes over and become stored: what the container does after the
63    /// flush that wrote them.
64    pub fn take_pending(&mut self) -> Option<Arc<Vec<u8>>> {
65        self.pending.take()
66    }
67}
68
69impl PartialEq for Blob {
70    /// Two stored values are equal (both mean "the file's bytes"); pending values compare by
71    /// content; a pending and a stored value differ, since one has an unflushed write.
72    fn eq(&self, other: &Self) -> bool {
73        match (&self.pending, &other.pending) {
74            (None, None) => true,
75            (Some(a), Some(b)) => a == b,
76            _ => false,
77        }
78    }
79}
80
81impl std::fmt::Debug for Blob {
82    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
83        match &self.pending {
84            Some(b) => write!(f, "Blob(pending {} bytes)", b.len()),
85            None => f.write_str("Blob(stored)"),
86        }
87    }
88}
89
90impl ColumnValue for Blob {
91    const SQL_TYPE: SqlType = SqlType::Blob;
92    /// Pending bytes go to the file; a stored value writes `NULL`, which the fold's upsert
93    /// and update statements treat as "leave the column alone".
94    fn to_sqlite_value(&self) -> Value {
95        match &self.pending {
96            Some(b) => Value::Blob(b.as_ref().clone()),
97            None => Value::Null,
98        }
99    }
100    /// A fault never reads the column, so whatever arrives decodes to the stored state.
101    fn from_sqlite_value(_: Value) -> Result<Self, DbError> {
102        Ok(Blob::stored())
103    }
104}
105
106// ---------------------------------------------------------------------------------------------
107// The accessor surface
108// ---------------------------------------------------------------------------------------------
109
110/// What a [`Blob`] field's accessor offers beyond the plain field: the bytes, on demand.
111///
112/// Every read is tracked through the field, so a view that shows the bytes re-renders when
113/// they are written. A pending value (written this turn, not yet flushed) answers from
114/// memory; otherwise the container answers from its byte-bounded memo or the file.
115pub trait BlobField {
116    /// The whole value. Empty when the column is `NULL` or the row is gone.
117    ///
118    /// # Errors
119    /// Returns the storage error when the value cannot be read.
120    fn bytes(&self) -> Result<Arc<Vec<u8>>, DbError>;
121    /// The value's length in bytes, without reading it (`length(col)` in SQL).
122    ///
123    /// # Errors
124    /// Returns the storage error when the length cannot be read.
125    fn len(&self) -> Result<usize, DbError>;
126    /// Whether the value is empty or `NULL`.
127    ///
128    /// # Errors
129    /// Returns the storage error when the length cannot be read.
130    fn is_empty(&self) -> Result<bool, DbError> {
131        Ok(self.len()? == 0)
132    }
133    /// `range` of the value, clamped to its length: `substr(col, …)` in SQL, so only the
134    /// requested bytes cross from the engine.
135    ///
136    /// # Errors
137    /// Returns the storage error when the bytes cannot be read.
138    fn read_range(&self, range: Range<usize>) -> Result<Vec<u8>, DbError>;
139    /// Write new bytes: the field becomes pending until the next flush.
140    fn write_bytes(&self, bytes: impl Into<Vec<u8>>);
141    /// A `std::io::Read` over the value, reading it in chunks through [`BlobField::read_range`].
142    fn reader(&self) -> BlobReader;
143}
144
145impl<S, M> BlobField for day_model::Field<S, M, Blob>
146where
147    S: day_model::Source<M> + Copy + 'static,
148    M: Model + 'static,
149{
150    fn bytes(&self) -> Result<Arc<Vec<u8>>, DbError> {
151        if let Some(pending) = day_reactive::Binding::read(self).pending {
152            return Ok(pending);
153        }
154        let (container, store, key) = locate(*self)?;
155        container.blob_bytes(store, key, column_of::<M>(self.label()))
156    }
157
158    fn len(&self) -> Result<usize, DbError> {
159        if let Some(pending) = day_reactive::Binding::read(self).pending {
160            return Ok(pending.len());
161        }
162        let (container, store, key) = locate(*self)?;
163        container.blob_len(store, key, column_of::<M>(self.label()))
164    }
165
166    fn read_range(&self, range: Range<usize>) -> Result<Vec<u8>, DbError> {
167        if let Some(pending) = day_reactive::Binding::read(self).pending {
168            return Ok(slice(&pending, range));
169        }
170        let (container, store, key) = locate(*self)?;
171        container.blob_range(store, key, column_of::<M>(self.label()), range)
172    }
173
174    fn write_bytes(&self, bytes: impl Into<Vec<u8>>) {
175        // The memo would otherwise keep answering with the old bytes until the flush.
176        if let Ok((container, store, key)) = locate(*self) {
177            container.blob_forget_field(store, key, column_of::<M>(self.label()));
178        }
179        day_reactive::Binding::write(self, Blob::new(bytes));
180    }
181
182    fn reader(&self) -> BlobReader {
183        let field = *self;
184        BlobReader {
185            read_range: Box::new(move |r| field.read_range(r)),
186            len: Box::new(move || field.len()),
187            pos: 0,
188            end: None,
189        }
190    }
191}
192
193type RangeReadFn = Box<dyn Fn(Range<usize>) -> Result<Vec<u8>, DbError>>;
194type LenFn = Box<dyn Fn() -> Result<usize, DbError>>;
195
196/// A `std::io::Read` over a lazy blob, in 256 KiB chunks.
197pub struct BlobReader {
198    read_range: RangeReadFn,
199    len: LenFn,
200    pos: usize,
201    end: Option<usize>,
202}
203
204impl BlobReader {
205    /// How much of the value one `read` asks the engine for at most.
206    pub const CHUNK: usize = 256 * 1024;
207}
208
209impl std::io::Read for BlobReader {
210    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
211        let end = if let Some(e) = self.end {
212            e
213        } else {
214            let e = (self.len)().map_err(std::io::Error::other)?;
215            self.end = Some(e);
216            e
217        };
218        if self.pos >= end || buf.is_empty() {
219            return Ok(0);
220        }
221        let want = buf.len().min(Self::CHUNK).min(end - self.pos);
222        let chunk = (self.read_range)(self.pos..self.pos + want).map_err(std::io::Error::other)?;
223        let n = chunk.len().min(buf.len());
224        buf[..n].copy_from_slice(&chunk[..n]);
225        self.pos += n;
226        Ok(n)
227    }
228}
229
230fn slice(bytes: &[u8], range: Range<usize>) -> Vec<u8> {
231    let start = range.start.min(bytes.len());
232    let end = range.end.clamp(start, bytes.len());
233    bytes[start..end].to_vec()
234}
235
236/// The stored column name of the field labelled `field` on `M`.
237fn column_of<M: Model>(field: &'static str) -> &'static str {
238    M::COLUMNS
239        .iter()
240        .find(|c| c.field == field)
241        .map_or(field, |c| c.name)
242}
243
244/// The container holding this field's store, with the store id and the row's key handle.
245fn locate<S, M>(field: day_model::Field<S, M, Blob>) -> Result<(ModelContainer, u64, u64), DbError>
246where
247    S: day_model::Source<M> + Copy + 'static,
248    M: 'static,
249{
250    let mut parts = Vec::new();
251    day_model::Source::components(field, &mut parts);
252    let (Some(&store), Some(&key)) = (parts.first(), parts.get(1)) else {
253        return Err(DbError::driver(
254            "a Blob field can only be read through a row of a container's store",
255        ));
256    };
257    crate::relations::container_for_store(store)
258        .map(|c| (c, store, key))
259        .ok_or_else(|| {
260            DbError::driver("a Blob field was read on a store no open ModelContainer holds")
261        })
262}
263
264// ---------------------------------------------------------------------------------------------
265// The memo
266// ---------------------------------------------------------------------------------------------
267
268/// Bytes read or written recently, bounded in bytes: the oldest entry leaves first, and a
269/// value larger than the whole limit is served but never kept.
270#[derive(Default)]
271pub(crate) struct BlobMemo {
272    entries: HashMap<(u64, u64, &'static str), Arc<Vec<u8>>>,
273    order: VecDeque<(u64, u64, &'static str)>,
274    bytes: usize,
275}
276
277impl BlobMemo {
278    pub(crate) fn get(&mut self, k: (u64, u64, &'static str)) -> Option<Arc<Vec<u8>>> {
279        let hit = self.entries.get(&k).cloned()?;
280        // Most recently used goes to the back.
281        if let Some(i) = self.order.iter().position(|e| *e == k) {
282            self.order.remove(i);
283            self.order.push_back(k);
284        }
285        Some(hit)
286    }
287
288    pub(crate) fn insert(&mut self, k: (u64, u64, &'static str), v: Arc<Vec<u8>>, limit: usize) {
289        self.forget_one(k);
290        if v.len() > limit {
291            return;
292        }
293        self.bytes += v.len();
294        self.entries.insert(k, v);
295        self.order.push_back(k);
296        while self.bytes > limit {
297            let Some(old) = self.order.pop_front() else {
298                break;
299            };
300            if let Some(v) = self.entries.remove(&old) {
301                self.bytes -= v.len();
302            }
303        }
304    }
305
306    pub(crate) fn forget_one(&mut self, k: (u64, u64, &'static str)) {
307        if let Some(v) = self.entries.remove(&k) {
308            self.bytes -= v.len();
309            if let Some(i) = self.order.iter().position(|e| *e == k) {
310                self.order.remove(i);
311            }
312        }
313    }
314
315    /// Drop every entry of one row (any column).
316    pub(crate) fn forget_row(&mut self, store: u64, key: u64) {
317        let keys: Vec<_> = self
318            .entries
319            .keys()
320            .filter(|(s, k, _)| *s == store && *k == key)
321            .copied()
322            .collect();
323        for k in keys {
324            self.forget_one(k);
325        }
326    }
327
328    /// Drop every entry of one store.
329    pub(crate) fn forget_store(&mut self, store: u64) {
330        let keys: Vec<_> = self
331            .entries
332            .keys()
333            .filter(|(s, _, _)| *s == store)
334            .copied()
335            .collect();
336        for k in keys {
337            self.forget_one(k);
338        }
339    }
340
341    pub(crate) fn clear(&mut self) {
342        self.entries.clear();
343        self.order.clear();
344        self.bytes = 0;
345    }
346
347    pub(crate) fn bytes(&self) -> usize {
348        self.bytes
349    }
350
351    pub(crate) fn len(&self) -> usize {
352        self.entries.len()
353    }
354}