use crate::events::{Building, Note, Present, Slot, Thread, Mark};
use rusqlite::{params, Connection, OptionalExtension};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use tokio::sync::broadcast;
pub const TABLES: &str = "
CREATE TABLE IF NOT EXISTS docs (
seq INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE,
org TEXT NOT NULL,
title TEXT NOT NULL,
text TEXT NOT NULL,
revision INTEGER NOT NULL,
created_at INTEGER NOT NULL,
created_by TEXT NOT NULL,
at INTEGER NOT NULL,
by TEXT NOT NULL,
state BLOB NOT NULL,
state_seq INTEGER NOT NULL,
deleted_at INTEGER
);
CREATE TABLE IF NOT EXISTS doc_log (
id TEXT NOT NULL,
seq INTEGER NOT NULL,
kind TEXT NOT NULL,
update_v1 BLOB,
title TEXT,
at INTEGER NOT NULL,
by TEXT NOT NULL,
principal TEXT NOT NULL,
bytes INTEGER NOT NULL,
authors TEXT,
PRIMARY KEY (id, seq)
);
";
pub const CHANNEL: usize = 4096;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Change {
Created(Note),
Updated { doc: Note, update: String, seq: u32, session: Option<String> },
Renamed(Note),
Deleted { id: String, org: String, at: i64, by: String },
Squashed { id: String, org: String, until: u32, removed: u32, at: i64, by: String },
Present { org: String, present: Present },
Left { id: String, org: String, session: String, client: Option<u64> },
Asked { org: String, thread: Thread },
Replied { org: String, thread: Thread },
Resolved { org: String, thread: Thread },
SlotChanged { org: String, slot: Slot },
BuildingChanged { org: String, slot: Slot },
PropsChanged { org: String, slot: Slot, by: String },
Marked { org: String, mark: Mark },
Unmarked { id: String, org: String, mark: String },
}
impl Change {
pub fn id(&self) -> &str {
match self {
Change::Created(d) | Change::Renamed(d) | Change::Updated { doc: d, .. } => &d.id,
Change::Deleted { id, .. } | Change::Squashed { id, .. } | Change::Left { id, .. } => id,
Change::Present { present, .. } => &present.id,
Change::Asked { thread, .. } | Change::Replied { thread, .. } | Change::Resolved { thread, .. } => &thread.id,
Change::SlotChanged { slot, .. } | Change::BuildingChanged { slot, .. } | Change::PropsChanged { slot, .. } => &slot.id,
Change::Marked { mark, .. } => &mark.id,
Change::Unmarked { id, .. } => id,
}
}
pub fn org(&self) -> &str {
match self {
Change::Created(d) | Change::Renamed(d) | Change::Updated { doc: d, .. } => &d.org,
Change::Deleted { org, .. } | Change::Squashed { org, .. } | Change::Present { org, .. } | Change::Left { org, .. } => org,
Change::Asked { org, .. } | Change::Replied { org, .. } | Change::Resolved { org, .. } => org,
Change::SlotChanged { org, .. } | Change::BuildingChanged { org, .. } | Change::PropsChanged { org, .. } => org,
Change::Marked { org, .. } | Change::Unmarked { org, .. } => org,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum StoreError {
NotFound(String),
NoRevision(String, u32),
NoThread(String),
NoMark(String),
NoComponent(String),
BadRange(String),
BadUpdate(String),
Quota(String),
Allowance(crate::quota::Refused),
TooLarge(String),
Db(String),
}
impl std::fmt::Display for StoreError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StoreError::NotFound(id) => write!(f, "no note {id}"),
StoreError::NoRevision(id, r) => write!(f, "note {id} has no change {r}"),
StoreError::NoThread(t) => write!(f, "no thread {t}"),
StoreError::NoMark(m) => write!(f, "no mark {m}"),
StoreError::NoComponent(m) => write!(f, "mark {m} places no component"),
StoreError::BadRange(e) => write!(f, "not a range of the text: {e}"),
StoreError::BadUpdate(e) => write!(f, "not a Yjs v1 update: {e}"),
StoreError::Quota(e) => write!(f, "{e}"),
StoreError::Allowance(r) => write!(f, "{}", r.message),
StoreError::TooLarge(e) => write!(f, "{e}"),
StoreError::Db(e) => write!(f, "the notes file: {e}"),
}
}
}
impl std::error::Error for StoreError {}
impl From<rusqlite::Error> for StoreError {
fn from(e: rusqlite::Error) -> Self {
StoreError::Db(e.to_string())
}
}
#[derive(Debug, Clone, Default)]
pub struct Listing {
pub orgs: Option<Vec<String>>,
pub query: Option<String>,
pub limit: usize,
pub after: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Page {
pub rows: Vec<Note>,
pub next: Option<String>,
pub remaining: u64,
}
pub struct NotesStore {
conn: Mutex<Connection>,
path: PathBuf,
changes: broadcast::Sender<Change>,
pub(crate) loaded: Mutex<HashMap<String, Arc<Mutex<yrs::Doc>>>>,
pub(crate) presence: crate::presence::Presence,
pub(crate) quota: std::sync::RwLock<Arc<dyn crate::quota::Quota>>,
pub(crate) builds: Mutex<HashMap<String, Building>>,
}
impl std::fmt::Debug for NotesStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("NotesStore").field("path", &self.path).finish()
}
}
pub(crate) fn now() -> i64 {
now_ms() / 1000
}
pub(crate) fn now_ms() -> i64 {
std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_millis() as i64)
}
pub(crate) const DOC_COLUMNS: &str = "id, org, title, text, revision, created_at, created_by, at, by";
pub(crate) fn doc_of(r: &rusqlite::Row<'_>) -> rusqlite::Result<Note> {
let body: String = r.get(3)?;
Ok(Note {
id: r.get(0)?,
org: r.get(1)?,
title: r.get(2)?,
bytes: body.len() as u64,
body: Some(body),
revision: r.get(4)?,
created_at: r.get(5)?,
created_by: r.get(6)?,
at: r.get(7)?,
by: r.get(8)?,
})
}
pub(crate) fn live_doc(conn: &Connection, id: &str) -> Result<Note, StoreError> {
conn.query_row(&format!("SELECT {DOC_COLUMNS} FROM docs WHERE id = ?1 AND deleted_at IS NULL"), [id], doc_of)
.optional()?
.ok_or_else(|| StoreError::NotFound(id.to_string()))
}
pub(crate) struct LogRow<'a> {
pub id: &'a str,
pub seq: u32,
pub kind: &'a str,
pub update: Option<&'a [u8]>,
pub title: Option<&'a str>,
pub at: i64,
pub by: &'a str,
pub principal: &'a str,
pub authors: Option<&'a str>,
}
pub(crate) fn log(tx: &Connection, row: LogRow<'_>) -> Result<(), StoreError> {
tx.execute(
"INSERT INTO doc_log (id, seq, kind, update_v1, title, at, by, principal, bytes, authors) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
params![row.id, row.seq, row.kind, row.update, row.title, row.at, row.by, row.principal, row.update.map_or(0, |u| u.len() as i64), row.authors],
)?;
Ok(())
}
impl NotesStore {
pub fn open(path: &Path) -> Result<Self, StoreError> {
if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
std::fs::create_dir_all(dir).map_err(|e| StoreError::Db(format!("{}: {e}", dir.display())))?;
}
let conn = Connection::open(path)?;
conn.execute_batch("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL;")?;
conn.execute_batch(TABLES)?;
conn.execute_batch(crate::threads::TABLES)?;
conn.execute_batch(crate::slot::TABLES)?;
conn.execute_batch(crate::marks::TABLES)?;
Ok(Self {
conn: Mutex::new(conn),
path: path.to_path_buf(),
changes: broadcast::channel(CHANNEL).0,
loaded: Mutex::new(HashMap::new()),
presence: crate::presence::Presence::default(),
quota: std::sync::RwLock::new(Arc::new(crate::quota::AllowAll)),
builds: Mutex::new(HashMap::new()),
})
}
pub fn metered(self, quota: Arc<dyn crate::quota::Quota>) -> Self {
self.set_quota(quota);
self
}
pub fn set_quota(&self, quota: Arc<dyn crate::quota::Quota>) {
*self.quota.write().unwrap_or_else(|p| p.into_inner()) = quota;
}
pub(crate) fn quota(&self) -> Arc<dyn crate::quota::Quota> {
self.quota.read().unwrap_or_else(|p| p.into_inner()).clone()
}
pub async fn usage_by_principal(self: &Arc<Self>) -> Result<Vec<(String, u64, u64)>, StoreError> {
let rows = self.usage_rows(false).await?;
Ok(rows.into_iter().map(|((p, _), (n, b))| (p, n, b)).collect())
}
pub async fn usage_daily(self: &Arc<Self>) -> Result<Vec<(String, i64, u64, u64)>, StoreError> {
let rows = self.usage_rows(true).await?;
Ok(rows.into_iter().map(|((p, d), (n, b))| (p, d, n, b)).collect())
}
async fn usage_rows(self: &Arc<Self>, by_day: bool) -> Result<std::collections::BTreeMap<(String, i64), (u64, u64)>, StoreError> {
self.blocking(move |s| {
let conn = s.lock();
let mut stmt = conn.prepare("SELECT kind, principal, bytes, authors, at FROM doc_log")?;
let rows = stmt.query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?, r.get::<_, i64>(2)?, r.get::<_, Option<String>>(3)?, r.get::<_, i64>(4)?)))?;
let mut out: std::collections::BTreeMap<(String, i64), (u64, u64)> = std::collections::BTreeMap::new();
for row in rows {
let (kind, principal, bytes, authors, at) = row?;
let day = if by_day { at.div_euclid(86_400_000) } else { 0 };
match (kind.as_str(), authors) {
("snapshot", Some(a)) => {
let authors: Vec<crate::events::Author> = serde_json::from_str(&a).unwrap_or_default();
if let Some(first) = authors.iter().min_by_key(|a| a.first) {
out.entry((first.principal.clone(), day)).or_default().0 += 1;
}
for (who, n) in crate::events::shares(bytes.max(0) as u64, &authors) {
out.entry((who, day)).or_default().1 += n;
}
}
(k, _) => {
let e = out.entry((principal, day)).or_default();
e.0 += u64::from(k == "create");
e.1 += bytes.max(0) as u64;
}
}
}
Ok(out)
})
.await
}
pub fn path(&self) -> &Path {
&self.path
}
pub(crate) fn lock(&self) -> std::sync::MutexGuard<'_, Connection> {
self.conn.lock().unwrap_or_else(|p| p.into_inner())
}
pub(crate) async fn blocking<T: Send + 'static>(self: &Arc<Self>, work: impl FnOnce(&Arc<NotesStore>) -> Result<T, StoreError> + Send + 'static) -> Result<T, StoreError> {
let store = Arc::clone(self);
tokio::task::spawn_blocking(move || work(&store)).await.map_err(|e| StoreError::Db(format!("notes task failed: {e}")))?
}
pub fn subscribe(&self) -> broadcast::Receiver<Change> {
self.changes.subscribe()
}
pub(crate) fn said(&self, change: Change) {
let _ = self.changes.send(change);
}
pub async fn get(self: &Arc<Self>, id: &str) -> Result<Note, StoreError> {
let id = id.to_string();
self.blocking(move |s| live_doc(&s.lock(), &id)).await
}
pub async fn list(self: &Arc<Self>, ask: Listing) -> Result<Page, StoreError> {
self.blocking(move |s| {
let conn = s.lock();
let mut sql = format!("SELECT {DOC_COLUMNS}, seq FROM docs WHERE deleted_at IS NULL");
let mut args: Vec<rusqlite::types::Value> = Vec::new();
if let Some(orgs) = &ask.orgs {
if orgs.is_empty() {
return Ok(Page { rows: Vec::new(), next: None, remaining: 0 });
}
sql.push_str(&format!(" AND org IN ({})", vec!["?"; orgs.len()].join(", ")));
args.extend(orgs.iter().cloned().map(rusqlite::types::Value::Text));
}
for word in ask.query.as_deref().unwrap_or_default().split_whitespace() {
sql.push_str(" AND instr(lower(title), ?) > 0");
args.push(rusqlite::types::Value::Text(word.to_lowercase()));
}
if let Some(after) = &ask.after {
let seq: Option<i64> = conn.query_row("SELECT seq FROM docs WHERE id = ?1", [after], |r| r.get(0)).optional()?;
sql.push_str(" AND seq < ?");
args.push(rusqlite::types::Value::Integer(seq.unwrap_or(i64::MAX)));
}
sql.push_str(" ORDER BY seq DESC");
let mut stmt = conn.prepare(&sql)?;
let all = stmt.query_map(rusqlite::params_from_iter(args.iter()), doc_of)?.collect::<Result<Vec<_>, _>>()?;
let limit = ask.limit.max(1);
let remaining = all.len().saturating_sub(limit) as u64;
let rows: Vec<Note> = all.into_iter().take(limit).map(|d| d.summary()).collect();
let next = (remaining > 0).then(|| rows.last().map(|d| d.id.clone())).flatten();
Ok(Page { rows, next, remaining })
})
.await
}
pub async fn rename(self: &Arc<Self>, id: &str, title: &str, by: &str, principal: &str) -> Result<Note, StoreError> {
let (id, title, by, principal) = (id.to_string(), title.to_string(), by.to_string(), principal.to_string());
let doc = self
.blocking(move |s| {
let conn = s.lock();
let tx = conn.unchecked_transaction()?;
let was = live_doc(&tx, &id)?;
let doc = Note { title, revision: was.revision + 1, at: now(), by, ..was };
tx.execute("UPDATE docs SET title = ?2, revision = ?3, at = ?4, by = ?5 WHERE id = ?1", params![doc.id, doc.title, doc.revision, doc.at, doc.by])?;
log(&tx, LogRow { id: &doc.id, seq: doc.revision, kind: "rename", update: None, title: Some(&doc.title), at: now_ms(), by: &doc.by, principal: &principal, authors: None })?;
tx.commit()?;
Ok(doc)
})
.await?;
self.said(Change::Renamed(doc.clone()));
Ok(doc)
}
pub async fn delete(self: &Arc<Self>, id: &str, by: &str, principal: &str) -> Result<(String, i64), StoreError> {
let (id, by, principal) = (id.to_string(), by.to_string(), principal.to_string());
let gone = id.clone();
let (org, at, by) = self
.blocking(move |s| {
let conn = s.lock();
let tx = conn.unchecked_transaction()?;
let was = live_doc(&tx, &id)?;
let doc = Note { revision: was.revision + 1, at: now(), by, ..was };
tx.execute("UPDATE docs SET deleted_at = ?2, revision = ?3, at = ?2, by = ?4 WHERE id = ?1", params![doc.id, doc.at, doc.revision, doc.by])?;
log(&tx, LogRow { id: &doc.id, seq: doc.revision, kind: "delete", update: None, title: None, at: now_ms(), by: &doc.by, principal: &principal, authors: None })?;
tx.commit()?;
s.loaded.lock().unwrap_or_else(|p| p.into_inner()).remove(&id);
Ok((doc.org, doc.at, doc.by))
})
.await?;
self.said(Change::Deleted { id: gone, org: org.clone(), at, by });
Ok((org, at))
}
pub async fn org_of(self: &Arc<Self>, id: &str) -> Result<String, StoreError> {
Ok(self.get(id).await?.org)
}
pub fn count(&self) -> usize {
self.lock().query_row("SELECT count(*) FROM docs WHERE deleted_at IS NULL", [], |r| r.get::<_, i64>(0)).unwrap_or(0) as usize
}
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_slot;