1use crate::events::{Author, Note, HistoryRow};
26use crate::quota::Reserved;
27use crate::store::{live_doc, log, now, now_ms, Change, NotesStore, LogRow, StoreError};
28use rusqlite::{params, Connection, OptionalExtension};
29use std::sync::{Arc, Mutex};
30use yrs::updates::decoder::Decode;
31use yrs::updates::encoder::Encode;
32use yrs::{GetString, OffsetKind, Options, ReadTxn, StateVector, Text, Transact, Update};
33
34pub const BODY: &str = "body";
36
37pub const SNAPSHOT_EVERY: u32 = 64;
39
40fn fresh() -> yrs::Doc {
42 yrs::Doc::with_options(Options { offset_kind: OffsetKind::Bytes, ..Options::default() })
43}
44
45pub(crate) fn text_of(doc: &yrs::Doc) -> String {
46 let body = doc.get_or_insert_text(BODY);
47 let txn = doc.transact();
48 body.get_string(&txn)
49}
50
51fn apply(doc: &yrs::Doc, bytes: &[u8]) -> Result<(), StoreError> {
52 let update = Update::decode_v1(bytes).map_err(|e| StoreError::BadUpdate(e.to_string()))?;
53 let mut txn = doc.transact_mut();
54 txn.apply_update(update).map_err(|e| StoreError::BadUpdate(e.to_string()))
55}
56
57pub(crate) fn b64(bytes: &[u8]) -> String {
58 use base64::Engine;
59 base64::engine::general_purpose::STANDARD.encode(bytes)
60}
61
62pub fn unb64(text: &str) -> Result<Vec<u8>, StoreError> {
64 use base64::Engine;
65 base64::engine::general_purpose::STANDARD.decode(text.trim()).map_err(|e| StoreError::BadUpdate(format!("not base64: {e}")))
66}
67
68fn rewrite(body: &yrs::TextRef, txn: &mut yrs::TransactionMut, old: &str, new: &str) {
72 for e in edits(old, new).iter().rev() {
73 if e.remove > 0 {
74 body.remove_range(txn, e.at as u32, e.remove as u32);
75 }
76 if !e.insert.is_empty() {
77 body.insert(txn, e.at as u32, &e.insert);
78 }
79 }
80}
81
82#[derive(Debug, Clone, PartialEq, Eq)]
84pub(crate) struct Edit {
85 at: usize,
86 remove: usize,
87 insert: String,
88}
89
90pub(crate) fn edits(old: &str, new: &str) -> Vec<Edit> {
101 let whole = || vec![Edit { at: 0, remove: old.len(), insert: new.to_string() }];
102 let diff = similar::TextDiff::configure().timeout(std::time::Duration::from_millis(250)).diff_chars(old, new);
103 let old_at: Vec<usize> = old.char_indices().map(|(i, _)| i).chain(std::iter::once(old.len())).collect();
104 let new_at: Vec<usize> = new.char_indices().map(|(i, _)| i).chain(std::iter::once(new.len())).collect();
105 let (mut o, mut n) = (0usize, 0usize);
107 let mut out = Vec::new();
108 for op in diff.ops() {
109 let (took, gave, same) = match *op {
110 similar::DiffOp::Equal { len, .. } => (len, len, true),
111 similar::DiffOp::Delete { old_len, .. } => (old_len, 0, false),
112 similar::DiffOp::Insert { new_len, .. } => (0, new_len, false),
113 similar::DiffOp::Replace { old_len, new_len, .. } => (old_len, new_len, false),
114 };
115 if o + took >= old_at.len() || n + gave >= new_at.len() {
116 return whole();
117 }
118 if !same {
119 out.push(Edit { at: old_at[o], remove: old_at[o + took] - old_at[o], insert: new[new_at[n]..new_at[n + gave]].to_string() });
120 }
121 o += took;
122 n += gave;
123 }
124 if made(old, &out) == new {
125 out
126 } else {
127 whole()
128 }
129}
130
131pub(crate) fn made(old: &str, edits: &[Edit]) -> String {
133 let mut text = old.to_string();
134 for e in edits.iter().rev() {
135 text.replace_range(e.at..e.at + e.remove, &e.insert);
136 }
137 text
138}
139
140#[derive(Debug, Clone, PartialEq, Eq)]
142pub struct Written {
143 pub doc: Note,
145 pub seq: Option<u32>,
147 pub stale: bool,
149}
150
151#[derive(Debug, Clone, PartialEq, Eq)]
153pub struct Synced {
154 pub update: String,
156 pub vector: String,
158 pub revision: u32,
159}
160
161fn replayed(conn: &Connection, id: &str, title_now: &str, seq: Option<u32>, at: Option<i64>) -> Result<(yrs::Doc, String, u32, i64, String), StoreError> {
166 let mut stmt = conn.prepare("SELECT seq, kind, update_v1, title, at, by FROM doc_log WHERE id = ?1 ORDER BY seq")?;
167 let rows = stmt.query_map([id], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, String>(1)?, r.get::<_, Option<Vec<u8>>>(2)?, r.get::<_, Option<String>>(3)?, r.get::<_, i64>(4)?, r.get::<_, String>(5)?)))?;
168 let crdt = fresh();
169 let mut last: Option<(u32, i64, String)> = None;
170 let mut title = title_now.to_string();
171 for row in rows {
172 let (n, kind, update, t, when, by) = row?;
173 if seq.is_some_and(|s| n > s) || at.is_some_and(|a| when > a) {
174 break;
175 }
176 if kind == "delete" {
177 continue;
178 }
179 if let Some(u) = update {
180 apply(&crdt, &u)?;
181 }
182 if let Some(t) = t {
183 title = t;
184 }
185 last = Some((n, when, by));
186 }
187 let missing = || StoreError::NoRevision(id.to_string(), seq.unwrap_or_default());
188 let (n, when, by) = last.ok_or_else(missing)?;
189 if seq.is_some_and(|s| s != n) && at.is_none() {
190 return Err(missing());
192 }
193 Ok((crdt, title, n, when, by))
194}
195
196impl NotesStore {
197 pub(crate) fn crdt(&self, conn: &Connection, id: &str) -> Result<Arc<Mutex<yrs::Doc>>, StoreError> {
200 if let Some(d) = self.loaded.lock().unwrap_or_else(|p| p.into_inner()).get(id) {
201 return Ok(Arc::clone(d));
202 }
203 let (state, state_seq): (Vec<u8>, u32) = conn
204 .query_row("SELECT state, state_seq FROM docs WHERE id = ?1 AND deleted_at IS NULL", [id], |r| Ok((r.get(0)?, r.get(1)?)))
205 .optional()?
206 .ok_or_else(|| StoreError::NotFound(id.to_string()))?;
207 let doc = fresh();
208 apply(&doc, &state)?;
209 let mut stmt = conn.prepare("SELECT update_v1 FROM doc_log WHERE id = ?1 AND seq > ?2 AND update_v1 IS NOT NULL ORDER BY seq")?;
210 for u in stmt.query_map(params![id, state_seq], |r| r.get::<_, Vec<u8>>(0))? {
211 apply(&doc, &u?)?;
212 }
213 let doc = Arc::new(Mutex::new(doc));
214 self.loaded.lock().unwrap_or_else(|p| p.into_inner()).insert(id.to_string(), Arc::clone(&doc));
215 Ok(doc)
216 }
217
218 fn forget(&self, id: &str) {
219 self.loaded.lock().unwrap_or_else(|p| p.into_inner()).remove(id);
220 }
221
222 pub async fn create(self: &Arc<Self>, org: &str, title: &str, body: &str, by: &str, principal: &str) -> Result<Note, StoreError> {
225 let (org, title, body, by, principal) = (org.to_string(), title.to_string(), body.to_string(), by.to_string(), principal.to_string());
226 let doc = self
227 .blocking(move |s| {
228 let quota = s.quota();
229 let born = quota.reserve_count(&principal).map_err(StoreError::Quota)?;
230 let crdt = fresh();
231 let update = {
232 let text = crdt.get_or_insert_text(BODY);
233 let mut txn = crdt.transact_mut();
234 text.insert(&mut txn, 0, &body);
235 txn.encode_update_v1()
236 };
237 let reserved = quota.reserve(&principal, update.len() as u64).map_err(StoreError::Quota)?;
239 let state = crdt.transact().encode_state_as_update_v1(&StateVector::default());
240 let conn = s.lock();
241 let tx = conn.unchecked_transaction()?;
242 let id = crate::vocab::mint(|id| tx.query_row("SELECT 1 FROM docs WHERE id = ?1", [id], |_| Ok(())).optional().ok().flatten().is_some());
243 let at = now();
244 let doc = Note { id, org, title, bytes: body.len() as u64, body: Some(body), revision: 1, created_at: at, created_by: by.clone(), at, by };
245 tx.execute(
246 "INSERT INTO docs (id, org, title, text, revision, created_at, created_by, at, by, state, state_seq) VALUES (?1, ?2, ?3, ?4, 1, ?5, ?6, ?5, ?6, ?7, 1)",
247 params![doc.id, doc.org, doc.title, doc.body.as_deref().unwrap_or_default(), doc.at, doc.by, state],
248 )?;
249 log(&tx, LogRow { id: &doc.id, seq: 1, kind: "create", update: Some(&update), title: Some(&doc.title), at: now_ms(), by: &doc.by, principal: &principal, authors: None })?;
250 tx.commit()?;
251 quota.commit(born);
252 quota.commit(reserved);
253 s.loaded.lock().unwrap_or_else(|p| p.into_inner()).insert(doc.id.clone(), Arc::new(Mutex::new(crdt)));
254 Ok(doc)
255 })
256 .await?;
257 self.said(Change::Created(doc.clone()));
258 Ok(doc)
259 }
260
261 fn append(&self, conn: &Connection, id: &str, crdt: &yrs::Doc, bytes: &[u8], kind: &str, by: &str, principal: &str) -> Result<(Note, u32), StoreError> {
267 let quota = self.quota();
268 let reserved: Reserved = quota.reserve(principal, bytes.len() as u64).map_err(StoreError::Quota)?;
269 let tx = conn.unchecked_transaction()?;
270 let was = live_doc(&tx, id)?;
271 let text = text_of(crdt);
272 let seq = was.revision + 1;
273 let doc = Note { bytes: text.len() as u64, body: Some(text), revision: seq, at: now(), by: by.to_string(), ..was };
274 tx.execute("UPDATE docs SET text = ?2, revision = ?3, at = ?4, by = ?5 WHERE id = ?1", params![doc.id, doc.body.as_deref().unwrap_or_default(), seq, doc.at, doc.by])?;
275 log(&tx, LogRow { id, seq, kind, update: Some(bytes), title: None, at: now_ms(), by, principal, authors: None })?;
276 let state_seq: u32 = tx.query_row("SELECT state_seq FROM docs WHERE id = ?1", [id], |r| r.get(0))?;
277 if seq - state_seq >= SNAPSHOT_EVERY {
278 let state = crdt.transact().encode_state_as_update_v1(&StateVector::default());
279 tx.execute("UPDATE docs SET state = ?2, state_seq = ?3 WHERE id = ?1", params![id, state, seq])?;
280 }
281 tx.commit()?;
282 quota.commit(reserved);
283 Ok((doc, seq))
284 }
285
286 pub async fn update(self: &Arc<Self>, id: &str, update: &str, by: &str, principal: &str, session: Option<String>) -> Result<Written, StoreError> {
290 let bytes = unb64(update)?;
291 let (id, by, principal) = (id.to_string(), by.to_string(), principal.to_string());
292 let said_update = update.trim().to_string();
293 let w = self
294 .blocking(move |s| {
295 let conn = s.lock();
296 let crdt = s.crdt(&conn, &id)?;
297 let crdt = crdt.lock().unwrap_or_else(|p| p.into_inner());
298 let written = apply(&crdt, &bytes).and_then(|()| s.append(&conn, &id, &crdt, &bytes, "update", &by, &principal));
299 match written {
300 Ok((doc, seq)) => Ok(Written { doc, seq: Some(seq), stale: false }),
301 Err(e) => {
302 drop(crdt);
303 s.forget(&id);
304 Err(e)
305 }
306 }
307 })
308 .await?;
309 self.said(Change::Updated { doc: w.doc.clone(), update: said_update, seq: w.seq.unwrap_or_default(), session });
310 Ok(w)
311 }
312
313 pub async fn save(self: &Arc<Self>, id: &str, body: &str, base: Option<u32>, by: &str, principal: &str) -> Result<Written, StoreError> {
320 let (id, body, by, principal) = (id.to_string(), body.to_string(), by.to_string(), principal.to_string());
321 let (w, update) = self
322 .blocking(move |s| {
323 let conn = s.lock();
324 let crdt = s.crdt(&conn, &id)?;
325 let crdt = crdt.lock().unwrap_or_else(|p| p.into_inner());
326 let was = live_doc(&conn, &id)?;
327 let stale = base.is_some_and(|b| b != was.revision);
328 if text_of(&crdt) == body {
329 return Ok((Written { doc: was, seq: None, stale }, None));
330 }
331 let at_base = base.filter(|_| stale).and_then(|b| replayed(&conn, &id, &was.title, Some(b), None).ok()).map(|(d, ..)| d);
335 let update = {
336 let on = at_base.as_ref().unwrap_or(&*crdt);
337 let old = text_of(on);
338 let text = on.get_or_insert_text(BODY);
339 let mut txn = on.transact_mut();
340 rewrite(&text, &mut txn, &old, &body);
341 txn.encode_update_v1()
342 };
343 if at_base.is_some() {
344 if let Err(e) = apply(&crdt, &update) {
345 drop(crdt);
346 s.forget(&id);
347 return Err(e);
348 }
349 }
350 match s.append(&conn, &id, &crdt, &update, "save", &by, &principal) {
351 Ok((doc, seq)) => Ok((Written { doc, seq: Some(seq), stale }, Some(update))),
352 Err(e) => {
353 drop(crdt);
354 s.forget(&id);
355 Err(e)
356 }
357 }
358 })
359 .await?;
360 if let (Some(update), Some(seq)) = (update, w.seq) {
361 self.said(Change::Updated { doc: w.doc.clone(), update: b64(&update), seq, session: None });
362 }
363 Ok(w)
364 }
365
366 pub async fn state(self: &Arc<Self>, id: &str, vector: Option<String>) -> Result<Synced, StoreError> {
369 let id = id.to_string();
370 self.blocking(move |s| {
371 let sv = match vector.as_deref().map(unb64).transpose()? {
372 Some(b) => StateVector::decode_v1(&b).map_err(|e| StoreError::BadUpdate(e.to_string()))?,
373 None => StateVector::default(),
374 };
375 let conn = s.lock();
376 let revision = live_doc(&conn, &id)?.revision;
377 let crdt = s.crdt(&conn, &id)?;
378 let crdt = crdt.lock().unwrap_or_else(|p| p.into_inner());
379 let txn = crdt.transact();
380 Ok(Synced { update: b64(&txn.encode_state_as_update_v1(&sv)), vector: b64(&txn.state_vector().encode_v1()), revision })
381 })
382 .await
383 }
384
385 pub async fn replay(self: &Arc<Self>, id: &str, seq: Option<u32>, at: Option<i64>) -> Result<Note, StoreError> {
391 let id = id.to_string();
392 self.blocking(move |s| {
393 let conn = s.lock();
394 let now = live_doc(&conn, &id)?;
395 let (crdt, title, n, when, by) = replayed(&conn, &id, &now.title, seq, at)?;
396 let text = text_of(&crdt);
397 Ok(Note { title, bytes: text.len() as u64, body: Some(text), revision: n, at: when / 1000, by, ..now })
398 })
399 .await
400 }
401
402 pub async fn restore(self: &Arc<Self>, id: &str, seq: u32, by: &str, principal: &str) -> Result<Written, StoreError> {
406 let then = self.replay(id, Some(seq), None).await?;
407 self.save(id, then.body.as_deref().unwrap_or_default(), None, by, principal).await
408 }
409
410 pub async fn history(self: &Arc<Self>, id: &str, limit: usize, before: Option<u32>) -> Result<Vec<HistoryRow>, StoreError> {
413 let id = id.to_string();
414 self.blocking(move |s| {
415 let conn = s.lock();
416 live_doc(&conn, &id)?;
417 let mut stmt = conn.prepare("SELECT seq, kind, at, by, principal, bytes, title, authors FROM doc_log WHERE id = ?1 AND seq < ?2 ORDER BY seq DESC LIMIT ?3")?;
418 let rows = stmt
419 .query_map(params![id, before.map_or(i64::MAX, i64::from), limit as i64], |r| {
420 let authors: Option<String> = r.get(7)?;
421 Ok(HistoryRow {
422 id: id.clone(),
423 seq: r.get(0)?,
424 kind: r.get(1)?,
425 at: r.get(2)?,
426 by: r.get(3)?,
427 principal: r.get(4)?,
428 bytes: r.get::<_, i64>(5)? as u64,
429 title: r.get(6)?,
430 authors: authors.and_then(|a| serde_json::from_str(&a).ok()),
431 })
432 })?
433 .collect::<Result<Vec<_>, _>>()?;
434 Ok(rows)
435 })
436 .await
437 }
438
439 pub async fn squash(self: &Arc<Self>, id: &str, until: u32, by: &str, principal: &str) -> Result<u32, StoreError> {
444 let (id, by, principal) = (id.to_string(), by.to_string(), principal.to_string());
445 let squasher = by.clone();
446 let then = self.replay(&id, Some(until), None).await?;
447 let (org, removed) = self
448 .blocking(move |s| {
449 let conn = s.lock();
450 let tx = conn.unchecked_transaction()?;
451 let doc = live_doc(&tx, &id)?;
452 let rows: Vec<(u32, String, Option<Vec<u8>>, i64, String, String, i64, Option<String>)> = {
453 let mut stmt = tx.prepare("SELECT seq, kind, update_v1, at, by, principal, bytes, authors FROM doc_log WHERE id = ?1 AND seq <= ?2 ORDER BY seq")?;
454 let rows = stmt
455 .query_map(params![id, until], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, r.get(6)?, r.get(7)?)))?
456 .collect::<Result<Vec<_>, _>>()?;
457 rows
458 };
459 if rows.len() < 2 {
460 return Ok((doc.org, 0));
461 }
462 let crdt = fresh();
463 let mut authors: Vec<Author> = Vec::new();
464 let mut add = |a: Author| match authors.iter_mut().find(|x| x.principal == a.principal) {
465 Some(x) => {
466 x.changes += a.changes;
467 x.bytes += a.bytes;
468 x.first = x.first.min(a.first);
469 x.last = x.last.max(a.last);
470 }
471 None => authors.push(a),
472 };
473 let mut last_at = 0;
474 for (_, kind, update, at, who, p, bytes, folded) in &rows {
475 if let Some(u) = update {
476 apply(&crdt, u)?;
477 }
478 last_at = *at;
479 match (kind.as_str(), folded) {
480 ("snapshot", Some(f)) => {
481 for a in serde_json::from_str::<Vec<Author>>(f).unwrap_or_default() {
482 add(a);
483 }
484 }
485 _ => add(Author { by: who.clone(), principal: p.clone(), changes: 1, bytes: *bytes as u64, first: *at, last: *at }),
486 }
487 }
488 let state = crdt.transact().encode_state_as_update_v1(&StateVector::default());
489 let authors = serde_json::to_string(&authors).map_err(|e| StoreError::Db(e.to_string()))?;
490 let mut moved: std::collections::BTreeMap<String, i64> = std::collections::BTreeMap::new();
493 for (_, kind, _, _, _, p, bytes, folded) in &rows {
494 match (kind.as_str(), folded) {
495 ("snapshot", Some(f)) => {
496 for (who, n) in crate::events::shares(*bytes as u64, &serde_json::from_str::<Vec<Author>>(f).unwrap_or_default()) {
497 *moved.entry(who).or_default() -= n as i64;
498 }
499 }
500 _ => *moved.entry(p.clone()).or_default() -= *bytes,
501 }
502 }
503 let folded: Vec<Author> = serde_json::from_str(&authors).unwrap_or_default();
504 for (who, n) in crate::events::shares(state.len() as u64, &folded) {
505 *moved.entry(who).or_default() += n as i64;
506 }
507 tx.execute("DELETE FROM doc_log WHERE id = ?1 AND seq <= ?2", params![id, until])?;
508 log(&tx, LogRow { id: &id, seq: until, kind: "snapshot", update: Some(&state), title: Some(&then.title), at: last_at, by: &by, principal: &principal, authors: Some(&authors) })?;
509 tx.commit()?;
510 let quota = s.quota();
511 for (who, delta) in moved.into_iter().filter(|(_, d)| *d != 0) {
512 quota.moved(&who, delta);
513 }
514 Ok((doc.org, rows.len() as u32 - 1))
515 })
516 .await?;
517 if removed > 0 {
518 self.said(Change::Squashed { id: then.id.clone(), org, until, removed, at: now(), by: squasher });
519 }
520 Ok(removed)
521 }
522}