Skip to main content

prolly_store_turso/
lib.rs

1//! Native Turso Database store adapter for `prolly-map`.
2
3use std::collections::HashMap;
4use std::fmt;
5use std::path::{Path, PathBuf};
6
7use prolly::{
8    BlockingRemoteProllyStore, NodePublication, NodePublicationHint, PublicationOrigin,
9    RemoteBatchOp, RemoteManifestUpdate, RemoteNamedRoot, RemoteRootCondition, RemoteRootWrite,
10    RemoteStoreBackend, RemoteTransactionConflict, RemoteTransactionUpdate,
11};
12use turso::transaction::TransactionBehavior;
13use turso::{Connection, Database, IntoParams};
14
15pub use prolly::{BlockingRemoteBuildError, BlockingRemoteStoreError, RemoteProllyStore};
16
17/// A complete async prolly store backed by native Turso Database.
18pub type TursoStore = RemoteProllyStore<TursoBackend>;
19
20/// Synchronous Turso store supporting `Prolly::indexed_map`.
21pub type SyncTursoStore = BlockingRemoteProllyStore<TursoBackend>;
22
23/// Native Turso backend used by [`TursoStore`].
24#[derive(Clone)]
25pub struct TursoBackend {
26    database: DatabaseHandle,
27}
28
29#[derive(Clone)]
30enum DatabaseHandle {
31    Local(Database),
32    #[cfg(feature = "turso-cloud-sync")]
33    Synced(turso::sync::Database),
34}
35
36impl TursoBackend {
37    /// Open or create a local Turso database and initialize the prolly schema.
38    pub async fn open(path: impl AsRef<Path>) -> Result<Self, TursoStoreError> {
39        let path = database_path(path.as_ref())?;
40        let database = turso::Builder::new_local(path).build().await?;
41        Self::from_local_database(database).await
42    }
43
44    /// Create a backend from a caller-configured local Turso database.
45    pub async fn from_local_database(database: Database) -> Result<Self, TursoStoreError> {
46        let backend = Self {
47            database: DatabaseHandle::Local(database),
48        };
49        backend.initialize_schema().await?;
50        Ok(backend)
51    }
52
53    /// Open a local replica connected to Turso Cloud.
54    ///
55    /// Store reads and writes remain local. Call [`TursoBackend::push`] and
56    /// [`TursoBackend::pull`] explicitly to synchronize with the remote.
57    #[cfg(feature = "turso-cloud-sync")]
58    pub async fn open_synced(
59        path: impl AsRef<Path>,
60        remote_url: impl Into<String>,
61        auth_token: impl Into<String>,
62    ) -> Result<Self, TursoStoreError> {
63        let path = database_path(path.as_ref())?;
64        let database = turso::sync::Builder::new_remote(path)
65            .with_remote_url(remote_url)
66            .with_auth_token(auth_token)
67            .build()
68            .await?;
69        Self::from_synced_database(database).await
70    }
71
72    /// Create a backend from a caller-configured synced Turso database.
73    #[cfg(feature = "turso-cloud-sync")]
74    pub async fn from_synced_database(
75        database: turso::sync::Database,
76    ) -> Result<Self, TursoStoreError> {
77        let backend = Self {
78            database: DatabaseHandle::Synced(database),
79        };
80        backend.initialize_schema().await?;
81        Ok(backend)
82    }
83
84    /// Return whether this backend has Turso Cloud synchronization configured.
85    pub fn is_synced(&self) -> bool {
86        match &self.database {
87            DatabaseHandle::Local(_) => false,
88            #[cfg(feature = "turso-cloud-sync")]
89            DatabaseHandle::Synced(_) => true,
90        }
91    }
92
93    /// Push locally committed changes to Turso Cloud.
94    #[cfg(feature = "turso-cloud-sync")]
95    pub async fn push(&self) -> Result<(), TursoStoreError> {
96        match &self.database {
97            DatabaseHandle::Synced(database) => Ok(database.push().await?),
98            DatabaseHandle::Local(_) => Err(TursoStoreError::NotSynced),
99        }
100    }
101
102    /// Pull and apply remote changes, returning whether any changes were applied.
103    #[cfg(feature = "turso-cloud-sync")]
104    pub async fn pull(&self) -> Result<bool, TursoStoreError> {
105        match &self.database {
106            DatabaseHandle::Synced(database) => Ok(database.pull().await?),
107            DatabaseHandle::Local(_) => Err(TursoStoreError::NotSynced),
108        }
109    }
110
111    async fn connect(&self) -> Result<Connection, TursoStoreError> {
112        match &self.database {
113            DatabaseHandle::Local(database) => Ok(database.connect()?),
114            #[cfg(feature = "turso-cloud-sync")]
115            DatabaseHandle::Synced(database) => Ok(database.connect().await?),
116        }
117    }
118
119    async fn initialize_schema(&self) -> Result<(), TursoStoreError> {
120        self.connect().await?.execute_batch(SCHEMA_SQL).await?;
121        Ok(())
122    }
123
124    async fn publish_node_entries(
125        &self,
126        publication: NodePublication<'_>,
127        behavior: TransactionBehavior,
128    ) -> Result<(), TursoStoreError> {
129        if publication.entries().is_empty() {
130            return Ok(());
131        }
132
133        let mut connection = self.connect().await?;
134        let transaction = connection.transaction_with_behavior(behavior).await?;
135        apply_node_entries(&transaction, publication.entries()).await?;
136        if let Some(hint) = publication.hint() {
137            transaction
138                .execute(
139                    UPSERT_HINT_SQL,
140                    (hint.namespace(), hint.key(), hint.value()),
141                )
142                .await?;
143        }
144        transaction.commit().await?;
145        Ok(())
146    }
147}
148
149impl fmt::Debug for TursoBackend {
150    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
151        formatter
152            .debug_struct("TursoBackend")
153            .field("is_synced", &self.is_synced())
154            .finish_non_exhaustive()
155    }
156}
157
158/// Errors returned by the native Turso backend.
159#[derive(Debug)]
160pub enum TursoStoreError {
161    /// Turso's string-based builder cannot represent the supplied path.
162    InvalidPath(PathBuf),
163    /// The native Turso engine rejected an operation.
164    Turso(turso::Error),
165    /// A cloud sync operation was requested on a local-only backend.
166    #[cfg(feature = "turso-cloud-sync")]
167    NotSynced,
168}
169
170impl fmt::Display for TursoStoreError {
171    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
172        match self {
173            Self::InvalidPath(path) => write!(
174                formatter,
175                "Turso database path is not valid UTF-8: {}",
176                path.display()
177            ),
178            Self::Turso(error) => write!(formatter, "Turso Database error: {error}"),
179            #[cfg(feature = "turso-cloud-sync")]
180            Self::NotSynced => formatter.write_str("Turso Cloud sync is not configured"),
181        }
182    }
183}
184
185impl std::error::Error for TursoStoreError {
186    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
187        match self {
188            Self::InvalidPath(_) => None,
189            Self::Turso(error) => Some(error),
190            #[cfg(feature = "turso-cloud-sync")]
191            Self::NotSynced => None,
192        }
193    }
194}
195
196impl RemoteStoreBackend for TursoBackend {
197    type Error = TursoStoreError;
198
199    async fn get_node(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
200        let connection = self.connect().await?;
201        query_optional_blob(&connection, SELECT_NODE_SQL, (key,)).await
202    }
203
204    async fn put_node(&self, key: &[u8], value: &[u8]) -> Result<(), Self::Error> {
205        let connection = self.connect().await?;
206        connection.execute(UPSERT_NODE_SQL, (key, value)).await?;
207        Ok(())
208    }
209
210    async fn delete_node(&self, key: &[u8]) -> Result<(), Self::Error> {
211        let connection = self.connect().await?;
212        connection.execute(DELETE_NODE_SQL, (key,)).await?;
213        Ok(())
214    }
215
216    async fn batch_nodes(&self, ops: &[RemoteBatchOp<'_>]) -> Result<(), Self::Error> {
217        let mut connection = self.connect().await?;
218        let transaction = connection
219            .transaction_with_behavior(TransactionBehavior::Immediate)
220            .await?;
221        apply_node_ops(&transaction, ops).await?;
222        transaction.commit().await?;
223        Ok(())
224    }
225
226    async fn batch_get_nodes_ordered(
227        &self,
228        keys: &[&[u8]],
229    ) -> Result<Vec<Option<Vec<u8>>>, Self::Error> {
230        if keys.is_empty() {
231            return Ok(Vec::new());
232        }
233        let connection = self.connect().await?;
234        let mut loaded = HashMap::with_capacity(keys.len());
235        for chunk in keys.chunks(MAX_SQL_BATCH_KEYS) {
236            let sql = select_nodes_sql(chunk.len());
237            let params = chunk.iter().map(|key| key.to_vec()).collect::<Vec<_>>();
238            let mut rows = connection.query(sql, params).await?;
239            while let Some(row) = rows.next().await? {
240                loaded.insert(row.get::<Vec<u8>>(0)?, row.get::<Vec<u8>>(1)?);
241            }
242        }
243        Ok(keys.iter().map(|key| loaded.get(*key).cloned()).collect())
244    }
245
246    fn prefers_batch_reads(&self) -> bool {
247        true
248    }
249
250    async fn batch_put_nodes(&self, entries: &[(&[u8], &[u8])]) -> Result<(), Self::Error> {
251        self.publish_node_entries(
252            NodePublication::new(entries, PublicationOrigin::General),
253            TransactionBehavior::Immediate,
254        )
255        .await
256    }
257
258    async fn list_node_cids(&self) -> Result<Vec<Vec<u8>>, Self::Error> {
259        let connection = self.connect().await?;
260        query_blob_column(&connection, SELECT_NODE_CIDS_SQL, ()).await
261    }
262
263    fn supports_hints(&self) -> bool {
264        true
265    }
266
267    fn prefers_rightmost_path_hints(&self) -> bool {
268        true
269    }
270
271    async fn get_hint(&self, namespace: &[u8], key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
272        let connection = self.connect().await?;
273        query_optional_blob(&connection, SELECT_HINT_SQL, (namespace, key)).await
274    }
275
276    async fn put_hint(
277        &self,
278        namespace: &[u8],
279        key: &[u8],
280        value: &[u8],
281    ) -> Result<(), Self::Error> {
282        let connection = self.connect().await?;
283        connection
284            .execute(UPSERT_HINT_SQL, (namespace, key, value))
285            .await?;
286        Ok(())
287    }
288
289    async fn batch_put_nodes_with_hint(
290        &self,
291        entries: &[(&[u8], &[u8])],
292        namespace: &[u8],
293        key: &[u8],
294        value: &[u8],
295    ) -> Result<(), Self::Error> {
296        self.publish_node_entries(
297            NodePublication::with_hint(
298                entries,
299                NodePublicationHint::new(namespace, key, value),
300                PublicationOrigin::General,
301            ),
302            TransactionBehavior::Immediate,
303        )
304        .await
305    }
306
307    async fn publish_nodes(&self, publication: NodePublication<'_>) -> Result<(), Self::Error> {
308        self.publish_node_entries(
309            publication,
310            publication_transaction_behavior(publication.origin()),
311        )
312        .await
313    }
314
315    async fn get_root_manifest(&self, name: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
316        let connection = self.connect().await?;
317        query_optional_blob(&connection, SELECT_ROOT_SQL, (name,)).await
318    }
319
320    async fn put_root_manifest(&self, name: &[u8], manifest: &[u8]) -> Result<(), Self::Error> {
321        let connection = self.connect().await?;
322        connection
323            .execute(UPSERT_ROOT_SQL, (name, manifest))
324            .await?;
325        Ok(())
326    }
327
328    async fn delete_root_manifest(&self, name: &[u8]) -> Result<(), Self::Error> {
329        let connection = self.connect().await?;
330        connection.execute(DELETE_ROOT_SQL, (name,)).await?;
331        Ok(())
332    }
333
334    async fn compare_and_swap_root_manifest(
335        &self,
336        name: &[u8],
337        expected: Option<&[u8]>,
338        new: Option<&[u8]>,
339    ) -> Result<RemoteManifestUpdate, Self::Error> {
340        let mut connection = self.connect().await?;
341        let transaction = connection
342            .transaction_with_behavior(TransactionBehavior::Immediate)
343            .await?;
344        let current = query_optional_blob(&transaction, SELECT_ROOT_SQL, (name,)).await?;
345
346        if current.as_deref() != expected {
347            transaction.rollback().await?;
348            return Ok(RemoteManifestUpdate::Conflict { current });
349        }
350
351        apply_root_write(&transaction, name, new).await?;
352        transaction.commit().await?;
353        Ok(RemoteManifestUpdate::Applied)
354    }
355
356    async fn list_root_manifests(&self) -> Result<Vec<RemoteNamedRoot>, Self::Error> {
357        let connection = self.connect().await?;
358        let mut rows = connection.query(SELECT_ROOTS_SQL, ()).await?;
359        let mut roots = Vec::new();
360        while let Some(row) = rows.next().await? {
361            roots.push(RemoteNamedRoot::new(row.get(0)?, row.get(1)?));
362        }
363        Ok(roots)
364    }
365
366    fn supports_transactions(&self) -> bool {
367        true
368    }
369
370    async fn commit_transaction(
371        &self,
372        node_writes: &[RemoteBatchOp<'_>],
373        root_conditions: &[RemoteRootCondition],
374        root_writes: &[RemoteRootWrite],
375    ) -> Result<RemoteTransactionUpdate, Self::Error> {
376        let mut connection = self.connect().await?;
377        let transaction = connection
378            .transaction_with_behavior(TransactionBehavior::Immediate)
379            .await?;
380
381        for condition in root_conditions {
382            let current =
383                query_optional_blob(&transaction, SELECT_ROOT_SQL, (condition.name.as_slice(),))
384                    .await?;
385            if current.as_deref() != condition.expected.as_deref() {
386                transaction.rollback().await?;
387                return Ok(RemoteTransactionUpdate::Conflict(
388                    RemoteTransactionConflict::new(
389                        condition.name.clone(),
390                        condition.expected.clone(),
391                        current,
392                    ),
393                ));
394            }
395        }
396
397        apply_node_ops(&transaction, node_writes).await?;
398        for write in root_writes {
399            match write {
400                RemoteRootWrite::Put { name, manifest } => {
401                    apply_root_write(&transaction, name, Some(manifest)).await?;
402                }
403                RemoteRootWrite::Delete { name } => {
404                    apply_root_write(&transaction, name, None).await?;
405                }
406            }
407        }
408
409        transaction.commit().await?;
410        Ok(RemoteTransactionUpdate::Applied)
411    }
412}
413
414fn publication_transaction_behavior(origin: PublicationOrigin) -> TransactionBehavior {
415    match origin {
416        PublicationOrigin::PointUpsert => TransactionBehavior::Deferred,
417        _ => TransactionBehavior::Immediate,
418    }
419}
420
421async fn query_optional_blob(
422    connection: &Connection,
423    sql: &str,
424    params: impl IntoParams,
425) -> Result<Option<Vec<u8>>, TursoStoreError> {
426    let mut rows = connection.query(sql, params).await?;
427    let value = rows.next().await?.map(|row| row.get(0)).transpose()?;
428    while rows.next().await?.is_some() {}
429    Ok(value)
430}
431
432async fn query_blob_column(
433    connection: &Connection,
434    sql: &str,
435    params: impl IntoParams,
436) -> Result<Vec<Vec<u8>>, TursoStoreError> {
437    let mut rows = connection.query(sql, params).await?;
438    let mut values = Vec::new();
439    while let Some(row) = rows.next().await? {
440        values.push(row.get(0)?);
441    }
442    Ok(values)
443}
444
445async fn apply_node_ops(
446    connection: &Connection,
447    ops: &[RemoteBatchOp<'_>],
448) -> Result<(), TursoStoreError> {
449    let mut upsert = connection.prepare(UPSERT_NODE_SQL).await?;
450    let mut delete = connection.prepare(DELETE_NODE_SQL).await?;
451    for op in ops {
452        match op {
453            RemoteBatchOp::Upsert { key, value } => {
454                upsert.execute((*key, *value)).await?;
455            }
456            RemoteBatchOp::Delete { key } => {
457                delete.execute((*key,)).await?;
458            }
459        }
460    }
461    Ok(())
462}
463
464async fn apply_node_entries(
465    connection: &Connection,
466    entries: &[(&[u8], &[u8])],
467) -> Result<(), TursoStoreError> {
468    for chunk in entries.chunks(MAX_SQL_BATCH_KEYS) {
469        let sql = upsert_nodes_sql(chunk.len());
470        let mut params = Vec::with_capacity(chunk.len() * 2);
471        for (key, value) in chunk {
472            params.push(key.to_vec());
473            params.push(value.to_vec());
474        }
475        connection.execute(sql, params).await?;
476    }
477    Ok(())
478}
479
480fn select_nodes_sql(count: usize) -> String {
481    let placeholders = (1..=count)
482        .map(|index| format!("?{index}"))
483        .collect::<Vec<_>>()
484        .join(", ");
485    format!("SELECT cid, node FROM prolly_nodes WHERE cid IN ({placeholders})")
486}
487
488fn upsert_nodes_sql(count: usize) -> String {
489    let values = (0..count)
490        .map(|index| format!("(?{}, ?{})", index * 2 + 1, index * 2 + 2))
491        .collect::<Vec<_>>()
492        .join(", ");
493    format!(
494        "INSERT INTO prolly_nodes (cid, node) VALUES {values} \
495         ON CONFLICT(cid) DO UPDATE SET node = excluded.node"
496    )
497}
498
499async fn apply_root_write(
500    connection: &Connection,
501    name: &[u8],
502    manifest: Option<&[u8]>,
503) -> Result<(), TursoStoreError> {
504    match manifest {
505        Some(manifest) => {
506            connection
507                .execute(UPSERT_ROOT_SQL, (name, manifest))
508                .await?;
509        }
510        None => {
511            connection.execute(DELETE_ROOT_SQL, (name,)).await?;
512        }
513    }
514    Ok(())
515}
516
517impl From<turso::Error> for TursoStoreError {
518    fn from(error: turso::Error) -> Self {
519        Self::Turso(error)
520    }
521}
522
523fn database_path(path: &Path) -> Result<&str, TursoStoreError> {
524    path.to_str()
525        .ok_or_else(|| TursoStoreError::InvalidPath(path.to_path_buf()))
526}
527
528const SCHEMA_SQL: &str = "\
529CREATE TABLE IF NOT EXISTS prolly_nodes (
530  cid BLOB PRIMARY KEY,
531  node BLOB NOT NULL
532);
533CREATE TABLE IF NOT EXISTS prolly_hints (
534  namespace BLOB NOT NULL,
535  key BLOB NOT NULL,
536  value BLOB NOT NULL,
537  PRIMARY KEY(namespace, key)
538);
539CREATE TABLE IF NOT EXISTS prolly_roots (
540  name BLOB PRIMARY KEY,
541  manifest BLOB NOT NULL
542);";
543
544const SELECT_NODE_SQL: &str = "SELECT node FROM prolly_nodes WHERE cid = ?1";
545const MAX_SQL_BATCH_KEYS: usize = 256;
546const UPSERT_NODE_SQL: &str = "\
547INSERT INTO prolly_nodes (cid, node) VALUES (?1, ?2)
548ON CONFLICT(cid) DO UPDATE SET node = excluded.node";
549const DELETE_NODE_SQL: &str = "DELETE FROM prolly_nodes WHERE cid = ?1";
550const SELECT_NODE_CIDS_SQL: &str = "SELECT cid FROM prolly_nodes ORDER BY cid";
551const SELECT_HINT_SQL: &str = "SELECT value FROM prolly_hints WHERE namespace = ?1 AND key = ?2";
552const UPSERT_HINT_SQL: &str = "\
553INSERT INTO prolly_hints (namespace, key, value) VALUES (?1, ?2, ?3)
554ON CONFLICT(namespace, key) DO UPDATE SET value = excluded.value";
555const SELECT_ROOT_SQL: &str = "SELECT manifest FROM prolly_roots WHERE name = ?1";
556const UPSERT_ROOT_SQL: &str = "\
557INSERT INTO prolly_roots (name, manifest) VALUES (?1, ?2)
558ON CONFLICT(name) DO UPDATE SET manifest = excluded.manifest";
559const DELETE_ROOT_SQL: &str = "DELETE FROM prolly_roots WHERE name = ?1";
560const SELECT_ROOTS_SQL: &str = "SELECT name, manifest FROM prolly_roots ORDER BY name";
561
562#[cfg(test)]
563mod tests {
564    use super::*;
565
566    #[test]
567    fn only_point_upserts_use_deferred_publication_transactions() {
568        assert!(matches!(
569            publication_transaction_behavior(PublicationOrigin::PointUpsert),
570            TransactionBehavior::Deferred
571        ));
572
573        for origin in [
574            PublicationOrigin::General,
575            PublicationOrigin::PointDelete,
576            PublicationOrigin::BatchMutation,
577            PublicationOrigin::TreeBuild,
578            PublicationOrigin::Merge,
579            PublicationOrigin::RangeDelete,
580            PublicationOrigin::Replication,
581            PublicationOrigin::Maintenance,
582        ] {
583            assert!(matches!(
584                publication_transaction_behavior(origin),
585                TransactionBehavior::Immediate
586            ));
587        }
588    }
589}