Skip to main content

menger_notes/
store.rs

1//! **The notes store** — the notes, in a SQLite file of the service's
2//! own, `notes.db`: never the tickets file (V2-1: the file is opened by the
3//! tickets service alone).
4//!
5//! A note's body is a **Yjs CRDT** (V2-29's `pairing`, the owner: *"live
6//! co editting … we should keep all changes over time too, so we could
7//! replay how it got built. with the ability to squash, permission
8//! gated"*). **Stored** in two tables:
9//!
10//! - `docs` — a row per note as it is now: its organization and title,
11//!   the markdown derived from its CRDT (`text`, what `get`, `list`, search,
12//!   agents and image references read), its revision (the last change's
13//!   `seq`), who made it and who wrote it last, and a snapshot of its CRDT
14//!   (`state` as of `state_seq`) so a load applies only what came after.
15//!   `deleted_at` set once deleted: it reads as none.
16//! - `doc_log` — **every change, forever until squashed**: a row per CRDT
17//!   update (`update`, the Yjs v1 bytes as the writer sent or the service
18//!   made them), per rename, per delete, with who (`by`, `principal`) and
19//!   when (`at`, unix milliseconds); `seq` counts them per note. A squash
20//!   ([`crate::crdt`]) replaces the rows up to a moment with one `snapshot`
21//!   row holding who wrote what as its `authors`.
22//!
23//! NOTES-5: and `note_marks`, a row per mark on a note's text ([`crate::marks`]).
24//!
25//! How the body is written, replayed and squashed is [`crate::crdt`]'s; who
26//! is on a note is [`crate::presence`]'s. Whether a caller may read or
27//! write a note is the gate's word on its organization, asked by the
28//! activation ([`crate::seam`]). Every write is said on the channel
29//! ([`NotesStore::subscribe`]) for `watch`.
30//!
31//! One connection behind a mutex; every read and write off the runtime.
32
33use crate::events::{Building, Note, Present, Slot, Thread, Mark};
34use rusqlite::{params, Connection, OptionalExtension};
35use std::collections::HashMap;
36use std::path::{Path, PathBuf};
37use std::sync::{Arc, Mutex};
38use tokio::sync::broadcast;
39
40/// The tables — their names kept from before NOTES-3, so the release before
41/// reads the same file. The table `docs`'s `seq` orders the list (newest made first) and is a
42/// page's cursor through its id.
43pub const TABLES: &str = "
44        CREATE TABLE IF NOT EXISTS docs (
45            seq         INTEGER PRIMARY KEY AUTOINCREMENT,
46            id          TEXT NOT NULL UNIQUE,
47            org         TEXT NOT NULL,
48            title       TEXT NOT NULL,
49            text        TEXT NOT NULL,
50            revision    INTEGER NOT NULL,
51            created_at  INTEGER NOT NULL,
52            created_by  TEXT NOT NULL,
53            at          INTEGER NOT NULL,
54            by          TEXT NOT NULL,
55            state       BLOB NOT NULL,
56            state_seq   INTEGER NOT NULL,
57            deleted_at  INTEGER
58        );
59        CREATE TABLE IF NOT EXISTS doc_log (
60            id         TEXT NOT NULL,
61            seq        INTEGER NOT NULL,
62            kind       TEXT NOT NULL,
63            update_v1  BLOB,
64            title      TEXT,
65            at         INTEGER NOT NULL,
66            by         TEXT NOT NULL,
67            principal  TEXT NOT NULL,
68            bytes      INTEGER NOT NULL,
69            authors    TEXT,
70            PRIMARY KEY (id, seq)
71        );
72    ";
73
74/// How many writes an open watch may fall behind before it hears `lagged`.
75pub const CHANNEL: usize = 4096;
76
77/// A write, as the channel says it to every open watch.
78#[derive(Debug, Clone, PartialEq, Eq)]
79pub enum Change {
80    /// A note made (body included).
81    Created(Note),
82    /// A CRDT update applied: the note after it (body included), the
83    /// update's bytes (base64), its `seq`, and the presence session it came
84    /// from, when it said.
85    Updated { doc: Note, update: String, seq: u32, session: Option<String> },
86    Renamed(Note),
87    Deleted { id: String, org: String, at: i64, by: String },
88    /// The changes up to `until` collapsed into one snapshot.
89    Squashed { id: String, org: String, until: u32, removed: u32, at: i64, by: String },
90    /// Someone is on a note, or moved their caret.
91    Present { org: String, present: Present },
92    /// Someone left a note.
93    Left { id: String, org: String, session: String, client: Option<u64> },
94    /// V2-30: a question asked on a range, a reply under it, a thread resolved (or opened again).
95    Asked { org: String, thread: Thread },
96    Replied { org: String, thread: Thread },
97    Resolved { org: String, thread: Thread },
98    /// NOTES-4: a slot filled or emptied, a build said, props merged (by `by`) — the slot whole.
99    SlotChanged { org: String, slot: Slot },
100    BuildingChanged { org: String, slot: Slot },
101    PropsChanged { org: String, slot: Slot, by: String },
102    /// NOTES-5: a mark made or changed (whole), or taken off.
103    Marked { org: String, mark: Mark },
104    Unmarked { id: String, org: String, mark: String },
105}
106
107impl Change {
108    /// The note it is of.
109    pub fn id(&self) -> &str {
110        match self {
111            Change::Created(d) | Change::Renamed(d) | Change::Updated { doc: d, .. } => &d.id,
112            Change::Deleted { id, .. } | Change::Squashed { id, .. } | Change::Left { id, .. } => id,
113            Change::Present { present, .. } => &present.id,
114            Change::Asked { thread, .. } | Change::Replied { thread, .. } | Change::Resolved { thread, .. } => &thread.id,
115            Change::SlotChanged { slot, .. } | Change::BuildingChanged { slot, .. } | Change::PropsChanged { slot, .. } => &slot.id,
116            Change::Marked { mark, .. } => &mark.id,
117            Change::Unmarked { id, .. } => id,
118        }
119    }
120
121    /// The organization that note is in.
122    pub fn org(&self) -> &str {
123        match self {
124            Change::Created(d) | Change::Renamed(d) | Change::Updated { doc: d, .. } => &d.org,
125            Change::Deleted { org, .. } | Change::Squashed { org, .. } | Change::Present { org, .. } | Change::Left { org, .. } => org,
126            Change::Asked { org, .. } | Change::Replied { org, .. } | Change::Resolved { org, .. } => org,
127            Change::SlotChanged { org, .. } | Change::BuildingChanged { org, .. } | Change::PropsChanged { org, .. } => org,
128            Change::Marked { org, .. } | Change::Unmarked { org, .. } => org,
129        }
130    }
131}
132
133/// Why the store did not.
134#[derive(Debug, Clone, PartialEq, Eq)]
135pub enum StoreError {
136    /// No note of that id (or a deleted one).
137    NotFound(String),
138    /// The note has no change at that `seq` (or it was squashed).
139    NoRevision(String, u32),
140    /// V2-30: no thread of that id on the note.
141    NoThread(String),
142    /// NOTES-5: no mark of that id on the note.
143    NoMark(String),
144    /// NOTES-5: the mark places no component, so it has no props.
145    NoComponent(String),
146    /// V2-30: a range that is not one of the text.
147    BadRange(String),
148    /// The update or state is not one the CRDT reads.
149    BadUpdate(String),
150    /// The quota refused the write (its words).
151    Quota(String),
152    /// V2-39: the quota refused it, with its reason (`over_allowance`, …).
153    Allowance(crate::quota::Refused),
154    /// NOTES-4: what a write would make is past its cap (its words).
155    TooLarge(String),
156    /// The file.
157    Db(String),
158}
159
160impl std::fmt::Display for StoreError {
161    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
162        match self {
163            StoreError::NotFound(id) => write!(f, "no note {id}"),
164            StoreError::NoRevision(id, r) => write!(f, "note {id} has no change {r}"),
165            StoreError::NoThread(t) => write!(f, "no thread {t}"),
166            StoreError::NoMark(m) => write!(f, "no mark {m}"),
167            StoreError::NoComponent(m) => write!(f, "mark {m} places no component"),
168            StoreError::BadRange(e) => write!(f, "not a range of the text: {e}"),
169            StoreError::BadUpdate(e) => write!(f, "not a Yjs v1 update: {e}"),
170            StoreError::Quota(e) => write!(f, "{e}"),
171            StoreError::Allowance(r) => write!(f, "{}", r.message),
172            StoreError::TooLarge(e) => write!(f, "{e}"),
173            StoreError::Db(e) => write!(f, "the notes file: {e}"),
174        }
175    }
176}
177
178impl std::error::Error for StoreError {}
179
180impl From<rusqlite::Error> for StoreError {
181    fn from(e: rusqlite::Error) -> Self {
182        StoreError::Db(e.to_string())
183    }
184}
185
186/// What a page of `list` asks.
187#[derive(Debug, Clone, Default)]
188pub struct Listing {
189    /// The organizations whose notes may be listed (`None`: every one).
190    pub orgs: Option<Vec<String>>,
191    /// Words the title holds, case-blind.
192    pub query: Option<String>,
193    /// The page's size.
194    pub limit: usize,
195    /// The id of the last row of the page before.
196    pub after: Option<String>,
197}
198
199/// A page of `list`: its rows, the cursor past it, how many more match.
200#[derive(Debug, Clone, PartialEq, Eq)]
201pub struct Page {
202    pub rows: Vec<Note>,
203    pub next: Option<String>,
204    pub remaining: u64,
205}
206
207/// The file, open, with the CRDTs it has loaded and who is on them.
208pub struct NotesStore {
209    conn: Mutex<Connection>,
210    path: PathBuf,
211    changes: broadcast::Sender<Change>,
212    /// Each loaded note's CRDT — loaded on its first write or sync, kept.
213    pub(crate) loaded: Mutex<HashMap<String, Arc<Mutex<yrs::Doc>>>>,
214    /// Who is on which note ([`crate::presence`]).
215    pub(crate) presence: crate::presence::Presence,
216    /// The meter every byte written is reserved against (V2-31's).
217    pub(crate) quota: std::sync::RwLock<Arc<dyn crate::quota::Quota>>,
218    /// NOTES-4: the build under way in each slot ([`crate::slot`]) — memory only.
219    pub(crate) builds: Mutex<HashMap<String, Building>>,
220}
221
222impl std::fmt::Debug for NotesStore {
223    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
224        f.debug_struct("NotesStore").field("path", &self.path).finish()
225    }
226}
227
228/// Now, in unix seconds.
229pub(crate) fn now() -> i64 {
230    now_ms() / 1000
231}
232
233/// Now, in unix milliseconds — a log row's `at`.
234pub(crate) fn now_ms() -> i64 {
235    std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_millis() as i64)
236}
237
238/// The columns a [`Note`] is read from, in [`doc_of`]'s order.
239pub(crate) const DOC_COLUMNS: &str = "id, org, title, text, revision, created_at, created_by, at, by";
240
241pub(crate) fn doc_of(r: &rusqlite::Row<'_>) -> rusqlite::Result<Note> {
242    let body: String = r.get(3)?;
243    Ok(Note {
244        id: r.get(0)?,
245        org: r.get(1)?,
246        title: r.get(2)?,
247        bytes: body.len() as u64,
248        body: Some(body),
249        revision: r.get(4)?,
250        created_at: r.get(5)?,
251        created_by: r.get(6)?,
252        at: r.get(7)?,
253        by: r.get(8)?,
254    })
255}
256
257/// The live note `id`, body and all, in `conn`.
258pub(crate) fn live_doc(conn: &Connection, id: &str) -> Result<Note, StoreError> {
259    conn.query_row(&format!("SELECT {DOC_COLUMNS} FROM docs WHERE id = ?1 AND deleted_at IS NULL"), [id], doc_of)
260        .optional()?
261        .ok_or_else(|| StoreError::NotFound(id.to_string()))
262}
263
264/// One row of the log, as it is written.
265pub(crate) struct LogRow<'a> {
266    pub id: &'a str,
267    pub seq: u32,
268    pub kind: &'a str,
269    pub update: Option<&'a [u8]>,
270    pub title: Option<&'a str>,
271    pub at: i64,
272    pub by: &'a str,
273    pub principal: &'a str,
274    pub authors: Option<&'a str>,
275}
276
277/// Appends `row` to the log.
278pub(crate) fn log(tx: &Connection, row: LogRow<'_>) -> Result<(), StoreError> {
279    tx.execute(
280        "INSERT INTO doc_log (id, seq, kind, update_v1, title, at, by, principal, bytes, authors) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
281        params![row.id, row.seq, row.kind, row.update, row.title, row.at, row.by, row.principal, row.update.map_or(0, |u| u.len() as i64), row.authors],
282    )?;
283    Ok(())
284}
285
286impl NotesStore {
287    /// The store on `path`, made if missing; a file without the tables
288    /// gains them, empty — once; on every later open a no-op. Every write
289    /// allowed ([`crate::quota::AllowAll`]) until a meter is set.
290    pub fn open(path: &Path) -> Result<Self, StoreError> {
291        if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
292            std::fs::create_dir_all(dir).map_err(|e| StoreError::Db(format!("{}: {e}", dir.display())))?;
293        }
294        let conn = Connection::open(path)?;
295        conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")?;
296        conn.execute_batch(TABLES)?;
297        conn.execute_batch(crate::threads::TABLES)?;
298        conn.execute_batch(crate::slot::TABLES)?;
299        conn.execute_batch(crate::marks::TABLES)?;
300        Ok(Self {
301            conn: Mutex::new(conn),
302            path: path.to_path_buf(),
303            changes: broadcast::channel(CHANNEL).0,
304            loaded: Mutex::new(HashMap::new()),
305            presence: crate::presence::Presence::default(),
306            quota: std::sync::RwLock::new(Arc::new(crate::quota::AllowAll)),
307            builds: Mutex::new(HashMap::new()),
308        })
309    }
310
311    /// The same store, its writes reserved against `quota` (V2-31's meter).
312    pub fn metered(self, quota: Arc<dyn crate::quota::Quota>) -> Self {
313        self.set_quota(quota);
314        self
315    }
316
317    /// V2-31: its writes reserved against `quota` from now on — the builder
318    /// sets the quota service's meter on a store already shared.
319    pub fn set_quota(&self, quota: Arc<dyn crate::quota::Quota>) {
320        *self.quota.write().unwrap_or_else(|p| p.into_inner()) = quota;
321    }
322
323    /// The meter now.
324    pub(crate) fn quota(&self) -> Arc<dyn crate::quota::Quota> {
325        self.quota.read().unwrap_or_else(|p| p.into_inner()).clone()
326    }
327
328    /// V2-31: what each principal keeps, as the quota service counts it —
329    /// **the update log as stored**: every change's bytes its writer's, and
330    /// a squash's snapshot shared among its authors by what each wrote
331    /// ([`crate::events::shares`]) — so a squash shrinks it; `count` the
332    /// notes they made (a `create` change, or, folded into a snapshot,
333    /// its earliest author's). (principal, count, bytes), by principal.
334    pub async fn usage_by_principal(self: &Arc<Self>) -> Result<Vec<(String, u64, u64)>, StoreError> {
335        let rows = self.usage_rows(false).await?;
336        Ok(rows.into_iter().map(|((p, _), (n, b))| (p, n, b)).collect())
337    }
338
339    /// V2-70: [`Self::usage_by_principal`] by the UTC day each change was
340    /// kept (days since 1970-01-01; a snapshot's shares on its own day):
341    /// (principal, day, count, bytes) — what the quota service's history is
342    /// backfilled from; summed per principal, the same.
343    pub async fn usage_daily(self: &Arc<Self>) -> Result<Vec<(String, i64, u64, u64)>, StoreError> {
344        let rows = self.usage_rows(true).await?;
345        Ok(rows.into_iter().map(|((p, d), (n, b))| (p, d, n, b)).collect())
346    }
347
348    /// What each principal keeps, by (principal, day — 0 unless `by_day`).
349    async fn usage_rows(self: &Arc<Self>, by_day: bool) -> Result<std::collections::BTreeMap<(String, i64), (u64, u64)>, StoreError> {
350        self.blocking(move |s| {
351            let conn = s.lock();
352            let mut stmt = conn.prepare("SELECT kind, principal, bytes, authors, at FROM doc_log")?;
353            let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?, r.get::<_, Option<String>>(3)?, r.get::<_, i64>(4)?)))?;
354            let mut out: std::collections::BTreeMap<(String, i64), (u64, u64)> = std::collections::BTreeMap::new();
355            for row in rows {
356                let (kind, principal, bytes, authors, at) = row?;
357                let day = if by_day { at.div_euclid(86_400_000) } else { 0 };
358                match (kind.as_str(), authors) {
359                    ("snapshot", Some(a)) => {
360                        let authors: Vec<crate::events::Author> = serde_json::from_str(&a).unwrap_or_default();
361                        if let Some(first) = authors.iter().min_by_key(|a| a.first) {
362                            out.entry((first.principal.clone(), day)).or_default().0 += 1;
363                        }
364                        for (who, n) in crate::events::shares(bytes.max(0) as u64, &authors) {
365                            out.entry((who, day)).or_default().1 += n;
366                        }
367                    }
368                    (k, _) => {
369                        let e = out.entry((principal, day)).or_default();
370                        e.0 += u64::from(k == "create");
371                        e.1 += bytes.max(0) as u64;
372                    }
373                }
374            }
375            Ok(out)
376        })
377        .await
378    }
379
380    /// The file it keeps.
381    pub fn path(&self) -> &Path {
382        &self.path
383    }
384
385    pub(crate) fn lock(&self) -> std::sync::MutexGuard<'_, Connection> {
386        self.conn.lock().unwrap_or_else(|p| p.into_inner())
387    }
388
389    /// Off the runtime, with the store.
390    pub(crate) async fn blocking<T: Send + 'static>(self: &Arc<Self>, work: impl FnOnce(&Arc<NotesStore>) -> Result<T, StoreError> + Send + 'static) -> Result<T, StoreError> {
391        let store = Arc::clone(self);
392        tokio::task::spawn_blocking(move || work(&store)).await.map_err(|e| StoreError::Db(format!("notes task failed: {e}")))?
393    }
394
395    /// Every write from now on, as it is committed.
396    pub fn subscribe(&self) -> broadcast::Receiver<Change> {
397        self.changes.subscribe()
398    }
399
400    /// Say `change` to every open watch.
401    pub(crate) fn said(&self, change: Change) {
402        let _ = self.changes.send(change);
403    }
404
405    /// The note `id` as it is now, body and all; [`StoreError::NotFound`]
406    /// for none (or a deleted one).
407    pub async fn get(self: &Arc<Self>, id: &str) -> Result<Note, StoreError> {
408        let id = id.to_string();
409        self.blocking(move |s| live_doc(&s.lock(), &id)).await
410    }
411
412    /// A page of the notes `ask` admits, newest made first, without
413    /// their bodies.
414    pub async fn list(self: &Arc<Self>, ask: Listing) -> Result<Page, StoreError> {
415        self.blocking(move |s| {
416            let conn = s.lock();
417            let mut sql = format!("SELECT {DOC_COLUMNS}, seq FROM docs WHERE deleted_at IS NULL");
418            let mut args: Vec<rusqlite::types::Value> = Vec::new();
419            if let Some(orgs) = &ask.orgs {
420                if orgs.is_empty() {
421                    return Ok(Page { rows: Vec::new(), next: None, remaining: 0 });
422                }
423                sql.push_str(&format!(" AND org IN ({})", vec!["?"; orgs.len()].join(", ")));
424                args.extend(orgs.iter().cloned().map(rusqlite::types::Value::Text));
425            }
426            for word in ask.query.as_deref().unwrap_or_default().split_whitespace() {
427                sql.push_str(" AND instr(lower(title), ?) > 0");
428                args.push(rusqlite::types::Value::Text(word.to_lowercase()));
429            }
430            if let Some(after) = &ask.after {
431                let seq: Option<i64> = conn.query_row("SELECT seq FROM docs WHERE id = ?1", [after], |r| r.get(0)).optional()?;
432                sql.push_str(" AND seq < ?");
433                args.push(rusqlite::types::Value::Integer(seq.unwrap_or(i64::MAX)));
434            }
435            sql.push_str(" ORDER BY seq DESC");
436            let mut stmt = conn.prepare(&sql)?;
437            let all = stmt.query_map(rusqlite::params_from_iter(args.iter()), doc_of)?.collect::<Result<Vec<_>, _>>()?;
438            let limit = ask.limit.max(1);
439            let remaining = all.len().saturating_sub(limit) as u64;
440            let rows: Vec<Note> = all.into_iter().take(limit).map(|d| d.summary()).collect();
441            let next = (remaining > 0).then(|| rows.last().map(|d| d.id.clone())).flatten();
442            Ok(Page { rows, next, remaining })
443        })
444        .await
445    }
446
447    /// Retitles `id` to `title`, by `by` (`principal`): a change more in the log.
448    pub async fn rename(self: &Arc<Self>, id: &str, title: &str, by: &str, principal: &str) -> Result<Note, StoreError> {
449        let (id, title, by, principal) = (id.to_string(), title.to_string(), by.to_string(), principal.to_string());
450        let doc = self
451            .blocking(move |s| {
452                let conn = s.lock();
453                let tx = conn.unchecked_transaction()?;
454                let was = live_doc(&tx, &id)?;
455                let doc = Note { title, revision: was.revision + 1, at: now(), by, ..was };
456                tx.execute("UPDATE docs SET title = ?2, revision = ?3, at = ?4, by = ?5 WHERE id = ?1", params![doc.id, doc.title, doc.revision, doc.at, doc.by])?;
457                log(&tx, LogRow { id: &doc.id, seq: doc.revision, kind: "rename", update: None, title: Some(&doc.title), at: now_ms(), by: &doc.by, principal: &principal, authors: None })?;
458                tx.commit()?;
459                Ok(doc)
460            })
461            .await?;
462        self.said(Change::Renamed(doc.clone()));
463        Ok(doc)
464    }
465
466    /// Deletes `id`, by `by`: it reads as none from now on; its log, and one
467    /// row more saying the delete, stay in the file.
468    pub async fn delete(self: &Arc<Self>, id: &str, by: &str, principal: &str) -> Result<(String, i64), StoreError> {
469        let (id, by, principal) = (id.to_string(), by.to_string(), principal.to_string());
470        let gone = id.clone();
471        let (org, at, by) = self
472            .blocking(move |s| {
473                let conn = s.lock();
474                let tx = conn.unchecked_transaction()?;
475                let was = live_doc(&tx, &id)?;
476                let doc = Note { revision: was.revision + 1, at: now(), by, ..was };
477                tx.execute("UPDATE docs SET deleted_at = ?2, revision = ?3, at = ?2, by = ?4 WHERE id = ?1", params![doc.id, doc.at, doc.revision, doc.by])?;
478                log(&tx, LogRow { id: &doc.id, seq: doc.revision, kind: "delete", update: None, title: None, at: now_ms(), by: &doc.by, principal: &principal, authors: None })?;
479                tx.commit()?;
480                s.loaded.lock().unwrap_or_else(|p| p.into_inner()).remove(&id);
481                Ok((doc.org, doc.at, doc.by))
482            })
483            .await?;
484        self.said(Change::Deleted { id: gone, org: org.clone(), at, by });
485        Ok((org, at))
486    }
487
488    /// The organization the live note `id` is in.
489    pub async fn org_of(self: &Arc<Self>, id: &str) -> Result<String, StoreError> {
490        Ok(self.get(id).await?.org)
491    }
492
493    /// How many notes the store keeps — for a log line.
494    pub fn count(&self) -> usize {
495        self.lock().query_row("SELECT count(*) FROM docs WHERE deleted_at IS NULL", [], |r| r.get::<_, i64>(0)).unwrap_or(0) as usize
496    }
497}
498
499#[cfg(test)]
500mod tests;
501#[cfg(test)]
502mod tests_slot;