1pub mod authz;
5pub mod backend;
6pub mod backup;
7pub mod lease;
8pub mod metrics;
9pub mod node;
10pub mod server;
11#[cfg(feature = "cljrs")]
12pub mod txfn;
13
14pub use backend::{LogBackend, NodeStore, StorageConnectionError, StoreSpec};
15
16use corium_core::{EntityId, IndexOrder, KeywordInterner, Partition, Schema};
17use corium_db::{Db, FIRST_USER_ID, Idents};
18use corium_index::Segment;
19use corium_log::{LogError, TransactionLog, TxRecord};
20use corium_store::{BlobStore, RootStore, StoreError};
21use corium_tx::{PreparedTx, TxError, TxItem, prepare};
22use std::{
23 sync::{Arc, Mutex, mpsc},
24 time::{SystemTime, UNIX_EPOCH},
25};
26use thiserror::Error;
27
28#[derive(Clone, Debug)]
30pub struct TxReport {
31 pub db_before: Db,
33 pub db_after: Db,
35 pub tx: PreparedTx,
37 pub tx_instant: i64,
39}
40
41#[derive(Debug, Error)]
43pub enum TransactError {
44 #[error(transparent)]
46 Tx(#[from] TxError),
47 #[error(transparent)]
49 Log(#[from] LogError),
50 #[error(transparent)]
52 Store(#[from] StoreError),
53 #[error("system clock is before Unix epoch")]
55 Clock,
56 #[error("index task failed: {0}")]
58 IndexTask(String),
59 #[error("an asynchronous transaction is already in progress")]
61 AsyncTransactionPending,
62 #[error("deposed: database root is owned by lease version {published}")]
64 Deposed {
65 published: u64,
67 },
68}
69
70fn seal_tx_instant(
83 datoms: &mut Vec<corium_core::Datom>,
84 t: u64,
85 now_ms: i64,
86 last_instant: i64,
87) -> Result<i64, TxError> {
88 match corium_db::bootstrap::asserted_instant(t, datoms) {
89 Some(supplied) if supplied <= last_instant => Err(TxError::TxInstantNotMonotonic {
90 supplied,
91 last: last_instant,
92 }),
93 Some(supplied) => Ok(supplied),
94 None => {
95 let instant = now_ms.max(last_instant.saturating_add(1));
96 datoms.push(corium_db::bootstrap::tx_instant_datom(t, instant));
97 Ok(instant)
98 }
99 }
100}
101
102fn effective_tx_instant(record: &TxRecord) -> i64 {
103 corium_db::bootstrap::asserted_instant(record.t, &record.datoms).unwrap_or(record.tx_instant)
104}
105
106struct State {
107 db: Db,
108 next_user: u64,
109 last_instant: i64,
110 subscribers: Vec<mpsc::Sender<TxReport>>,
111 async_pending: bool,
112}
113
114struct AsyncPending<'a> {
115 state: &'a Mutex<State>,
116 active: bool,
117}
118
119impl Drop for AsyncPending<'_> {
120 fn drop(&mut self) {
121 if self.active {
122 self.state
123 .lock()
124 .unwrap_or_else(std::sync::PoisonError::into_inner)
125 .async_pending = false;
126 }
127 }
128}
129
130fn next_user_id<'a>(datoms: impl Iterator<Item = &'a corium_core::Datom>, floor: u64) -> u64 {
133 datoms
134 .filter(|d| d.e.partition() == Partition::User as u32)
135 .map(|d| d.e.sequence() + 1)
136 .fold(floor, u64::max)
137}
138
139pub struct Prepared {
143 pub record: TxRecord,
145 db_before: Db,
146 db_after: Db,
147 tx: PreparedTx,
148 tx_instant: i64,
149}
150
151pub struct BatchCursor {
160 db: Db,
161 next_user: u64,
162 last_instant: i64,
163}
164
165impl BatchCursor {
166 #[must_use]
169 pub fn db(&self) -> &Db {
170 &self.db
171 }
172
173 pub fn prepare(
182 &mut self,
183 items: impl IntoIterator<Item = TxItem>,
184 now_ms: i64,
185 ) -> Result<Prepared, TxError> {
186 let before = self.db.clone();
187 let t = before.basis_t() + 1;
188 let tx_id = EntityId::new(Partition::Tx as u32, t);
189 let mut prepared = prepare(&before, items, tx_id, self.next_user)?;
190 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, now_ms, self.last_instant)?;
191 let after = before
192 .clone()
193 .with_transaction_at(t, tx_instant, &prepared.datoms);
194 self.next_user = prepared
195 .tempids
196 .values()
197 .filter(|e| e.partition() == Partition::User as u32)
198 .map(|e| e.sequence() + 1)
199 .max()
200 .unwrap_or(self.next_user)
201 .max(self.next_user);
202 self.last_instant = tx_instant;
203 self.db = after.clone();
204 Ok(Prepared {
205 record: TxRecord {
206 t,
207 tx_instant,
208 datoms: prepared.datoms.clone(),
209 },
210 db_before: before,
211 db_after: after,
212 tx: prepared,
213 tx_instant,
214 })
215 }
216}
217
218pub struct EmbeddedTransactor {
220 log: Arc<dyn TransactionLog>,
221 state: Mutex<State>,
222 async_commit: tokio::sync::Mutex<()>,
223}
224impl EmbeddedTransactor {
225 pub fn recover(schema: Schema, log: Arc<dyn TransactionLog>) -> Result<Self, TransactError> {
230 Self::recover_from(Db::new(schema), log)
231 }
232
233 pub fn recover_from(base: Db, log: Arc<dyn TransactionLog>) -> Result<Self, TransactError> {
239 let mut db = base;
240 let mut last_instant = i64::MIN;
241 for record in log.replay()? {
242 let tx_instant = effective_tx_instant(&record);
243 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
244 last_instant = last_instant.max(tx_instant);
245 }
246 let next_user = next_user_id(db.recorded_datoms(), FIRST_USER_ID);
250 Ok(Self {
251 log,
252 state: Mutex::new(State {
253 db,
254 next_user,
255 last_instant,
256 subscribers: Vec::new(),
257 async_pending: false,
258 }),
259 async_commit: tokio::sync::Mutex::new(()),
260 })
261 }
262
263 pub async fn recover_from_async(
268 base: Db,
269 log: Arc<dyn TransactionLog>,
270 ) -> Result<Self, TransactError> {
271 let records = log.replay_async().await?;
272 Ok(Self::recover_from_records(base, log, records))
273 }
274
275 fn recover_from_records(
276 mut db: Db,
277 log: Arc<dyn TransactionLog>,
278 records: Vec<TxRecord>,
279 ) -> Self {
280 let mut last_instant = i64::MIN;
281 for record in records {
282 let tx_instant = effective_tx_instant(&record);
283 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
284 last_instant = last_instant.max(tx_instant);
285 }
286 let next_user = next_user_id(db.recorded_datoms(), FIRST_USER_ID);
287 Self {
288 log,
289 state: Mutex::new(State {
290 db,
291 next_user,
292 last_instant,
293 subscribers: Vec::new(),
294 async_pending: false,
295 }),
296 async_commit: tokio::sync::Mutex::new(()),
297 }
298 }
299
300 pub fn recover_from_snapshot(
325 snapshot: Db,
326 next_entity_id: u64,
327 last_tx_instant: i64,
328 log: Arc<dyn TransactionLog>,
329 ) -> Result<Self, TransactError> {
330 let mut db = snapshot;
331 let index_basis = db.basis_t();
332 let mut last_instant = last_tx_instant;
333 let mut next_user = next_entity_id.max(FIRST_USER_ID);
336 for record in log.tx_range(index_basis + 1, None)? {
337 let tx_instant = effective_tx_instant(&record);
338 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
339 last_instant = last_instant.max(tx_instant);
340 next_user = next_user.max(next_user_id(record.datoms.iter(), next_user));
341 }
342 Ok(Self {
343 log,
344 state: Mutex::new(State {
345 db,
346 next_user,
347 last_instant,
348 subscribers: Vec::new(),
349 async_pending: false,
350 }),
351 async_commit: tokio::sync::Mutex::new(()),
352 })
353 }
354
355 pub async fn recover_from_snapshot_async(
361 snapshot: Db,
362 next_entity_id: u64,
363 last_tx_instant: i64,
364 log: Arc<dyn TransactionLog>,
365 ) -> Result<Self, TransactError> {
366 let index_basis = snapshot.basis_t();
367 let records = log.tx_range_async(index_basis + 1, None).await?;
368 let mut db = snapshot;
369 let mut last_instant = last_tx_instant;
370 let mut next_user = next_entity_id.max(FIRST_USER_ID);
371 for record in records {
372 let tx_instant = effective_tx_instant(&record);
373 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
374 last_instant = last_instant.max(tx_instant);
375 next_user = next_user.max(next_user_id(record.datoms.iter(), next_user));
376 }
377 Ok(Self {
378 log,
379 state: Mutex::new(State {
380 db,
381 next_user,
382 last_instant,
383 subscribers: Vec::new(),
384 async_pending: false,
385 }),
386 async_commit: tokio::sync::Mutex::new(()),
387 })
388 }
389
390 fn recovery_snapshot(&self) -> (Db, u64, i64) {
394 let state = self
395 .state
396 .lock()
397 .unwrap_or_else(std::sync::PoisonError::into_inner);
398 (state.db.clone(), state.next_user, state.last_instant)
399 }
400 #[must_use]
402 pub fn db(&self) -> Db {
403 self.state
404 .lock()
405 .unwrap_or_else(std::sync::PoisonError::into_inner)
406 .db
407 .clone()
408 }
409 pub fn subscribe(&self) -> mpsc::Receiver<TxReport> {
411 let (tx, rx) = mpsc::channel();
412 self.state
413 .lock()
414 .unwrap_or_else(std::sync::PoisonError::into_inner)
415 .subscribers
416 .push(tx);
417 rx
418 }
419 pub fn transact(
425 &self,
426 items: impl IntoIterator<Item = TxItem>,
427 ) -> Result<TxReport, TransactError> {
428 let mut state = self
429 .state
430 .lock()
431 .unwrap_or_else(std::sync::PoisonError::into_inner);
432 if state.async_pending {
433 return Err(TransactError::AsyncTransactionPending);
434 }
435 let before = state.db.clone();
436 let t = before.basis_t() + 1;
437 let tx_id = EntityId::new(Partition::Tx as u32, t);
438 let mut prepared = prepare(&before, items, tx_id, state.next_user)?;
439 let millis = i64::try_from(
440 SystemTime::now()
441 .duration_since(UNIX_EPOCH)
442 .map_err(|_| TransactError::Clock)?
443 .as_millis(),
444 )
445 .unwrap_or(i64::MAX);
446 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, millis, state.last_instant)?;
447 self.log.append(&TxRecord {
448 t,
449 tx_instant,
450 datoms: prepared.datoms.clone(),
451 })?;
452 state.db = before.with_transaction_at(t, tx_instant, &prepared.datoms);
453 state.last_instant = tx_instant;
454 state.next_user = prepared
455 .tempids
456 .values()
457 .filter(|e| e.partition() == Partition::User as u32)
458 .map(|e| e.sequence() + 1)
459 .max()
460 .unwrap_or(state.next_user)
461 .max(state.next_user);
462 let report = TxReport {
463 db_before: before,
464 db_after: state.db.clone(),
465 tx: prepared,
466 tx_instant,
467 };
468 state
469 .subscribers
470 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
471 Ok(report)
472 }
473
474 pub async fn transact_async(
483 &self,
484 items: impl IntoIterator<Item = TxItem>,
485 ) -> Result<TxReport, TransactError> {
486 let _commit = self.async_commit.lock().await;
487 let (before, prepared, t, tx_instant) = {
488 let mut state = self
489 .state
490 .lock()
491 .unwrap_or_else(std::sync::PoisonError::into_inner);
492 if state.async_pending {
493 return Err(TransactError::AsyncTransactionPending);
494 }
495 let before = state.db.clone();
496 let t = before.basis_t() + 1;
497 let tx_id = EntityId::new(Partition::Tx as u32, t);
498 let mut prepared = prepare(&before, items, tx_id, state.next_user)?;
499 let millis = i64::try_from(
500 SystemTime::now()
501 .duration_since(UNIX_EPOCH)
502 .map_err(|_| TransactError::Clock)?
503 .as_millis(),
504 )
505 .unwrap_or(i64::MAX);
506 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, millis, state.last_instant)?;
507 state.async_pending = true;
508 (before, prepared, t, tx_instant)
509 };
510 let record = TxRecord {
511 t,
512 tx_instant,
513 datoms: prepared.datoms.clone(),
514 };
515 let mut pending = AsyncPending {
516 state: &self.state,
517 active: true,
518 };
519 self.log.append_async(&record).await?;
520 let mut state = self
521 .state
522 .lock()
523 .unwrap_or_else(std::sync::PoisonError::into_inner);
524 debug_assert_eq!(state.db.basis_t() + 1, t);
525 state.db = state
529 .db
530 .clone()
531 .with_transaction_at(t, tx_instant, &prepared.datoms);
532 state.last_instant = tx_instant;
533 state.next_user = prepared
534 .tempids
535 .values()
536 .filter(|e| e.partition() == Partition::User as u32)
537 .map(|e| e.sequence() + 1)
538 .max()
539 .unwrap_or(state.next_user)
540 .max(state.next_user);
541 state.async_pending = false;
542 pending.active = false;
543 let report = TxReport {
544 db_before: before,
545 db_after: state.db.clone(),
546 tx: prepared,
547 tx_instant,
548 };
549 state
550 .subscribers
551 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
552 Ok(report)
553 }
554 #[must_use]
558 pub fn batch_cursor(&self) -> BatchCursor {
559 let state = self
560 .state
561 .lock()
562 .unwrap_or_else(std::sync::PoisonError::into_inner);
563 BatchCursor {
564 db: state.db.clone(),
565 next_user: state.next_user,
566 last_instant: state.last_instant,
567 }
568 }
569
570 pub fn install_batch(&self, cursor: BatchCursor, prepared: Vec<Prepared>) -> Vec<TxReport> {
578 let mut state = self
579 .state
580 .lock()
581 .unwrap_or_else(std::sync::PoisonError::into_inner);
582 state.db = cursor.db;
583 state.next_user = state.next_user.max(cursor.next_user);
584 state.last_instant = state.last_instant.max(cursor.last_instant);
585 let reports: Vec<TxReport> = prepared
586 .into_iter()
587 .map(|p| TxReport {
588 db_before: p.db_before,
589 db_after: p.db_after,
590 tx: p.tx,
591 tx_instant: p.tx_instant,
592 })
593 .collect();
594 for report in &reports {
595 state
596 .subscribers
597 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
598 }
599 reports
600 }
601
602 pub fn update_naming(&self, idents: Idents, interner: KeywordInterner) {
605 let mut state = self
606 .state
607 .lock()
608 .unwrap_or_else(std::sync::PoisonError::into_inner);
609 state.db = state.db.clone().with_naming(idents, interner);
610 }
611
612 pub async fn publish_indexes(
633 &self,
634 store: &(impl BlobStore + RootStore),
635 root_name: &str,
636 lease_version: u64,
637 ) -> Result<DbRoot, TransactError> {
638 let (snapshot, next_entity_id, last_tx_instant) = self.recovery_snapshot();
639 let datoms = snapshot.datoms();
640 let chunked = tokio::task::spawn_blocking(move || {
641 [
642 IndexOrder::Eavt,
643 IndexOrder::Aevt,
644 IndexOrder::Avet,
645 IndexOrder::Vaet,
646 ]
647 .into_iter()
648 .map(|order| {
649 let segment = Segment::build(order, datoms.clone());
650 corium_store::chunk_segment_keys(segment.entries().map(|(key, _)| key.as_slice()))
651 })
652 .collect::<Vec<_>>()
653 })
654 .await
655 .map_err(|error| TransactError::IndexTask(error.to_string()))?;
656 let mut ids = Vec::new();
657 for chunks in chunked {
658 let mut children = Vec::new();
659 for chunk in chunks {
660 children.push(store.put_if_absent(&chunk).await?);
661 }
662 let manifest = corium_store::encode_index_manifest(&children);
663 ids.push(store.put_if_absent(&manifest).await?);
664 }
665 let root = DbRoot {
666 format_version: corium_store::FORMAT_VERSION,
667 lease_version,
668 owner: String::new(),
669 lease_expires_unix_ms: 0,
670 owner_endpoint: String::new(),
671 index_basis_t: snapshot.basis_t(),
672 roots: Some([
673 ids[0].clone(),
674 ids[1].clone(),
675 ids[2].clone(),
676 ids[3].clone(),
677 ]),
678 next_entity_id,
680 last_tx_instant,
681 };
682 publish_root(store, root_name, &root).await?;
683 Ok(root)
684 }
685}
686
687pub async fn publish_root(
694 store: &dyn RootStore,
695 root_name: &str,
696 root: &DbRoot,
697) -> Result<(), TransactError> {
698 loop {
699 let previous = store.get_root(root_name).await?;
700 let stored = previous.as_deref().and_then(DbRoot::decode);
701 let mut next = root.clone();
702 if let Some(stored) = stored {
703 if stored.lease_version > root.lease_version {
704 return Err(TransactError::Deposed {
705 published: stored.lease_version,
706 });
707 }
708 if stored.lease_version == root.lease_version
709 && stored.index_basis_t >= root.index_basis_t
710 {
711 return Ok(());
712 }
713 if stored.lease_version == root.lease_version {
716 next.owner = stored.owner;
717 next.lease_expires_unix_ms = stored.lease_expires_unix_ms;
718 next.owner_endpoint = stored.owner_endpoint;
719 }
720 }
721 match store
722 .cas_root(root_name, previous.as_deref(), &next.encode())
723 .await
724 {
725 Ok(()) => return Ok(()),
726 Err(StoreError::CasFailed { .. }) => {}
727 Err(error) => return Err(error.into()),
728 }
729 }
730}
731
732pub use corium_store::{DbRoot, db_root_name};