1pub mod authz;
5pub mod backend;
6pub mod backup;
7pub mod lease;
8pub mod metrics;
9pub mod node;
10pub mod server;
11#[cfg(feature = "cljrs")]
12pub mod txfn;
13
14pub use backend::{LogBackend, NodeStore, StorageConnectionError, StorageInfoConfig, StoreSpec};
15#[cfg(feature = "s3")]
16pub use backend::{S3ReadOnlyConfig, S3ReadOnlyCredentials};
17
18use corium_core::{EntityId, IndexOrder, KeywordInterner, Partition, Schema};
19use corium_db::{Db, FIRST_USER_ID, Idents};
20use corium_index::Segment;
21use corium_log::{LogError, TransactionLog, TxRecord};
22use corium_store::{BlobStore, RootStore, StoreError};
23use corium_tx::{PreparedTx, TxError, TxItem, prepare};
24use std::{
25 sync::{Arc, Mutex, mpsc},
26 time::{SystemTime, UNIX_EPOCH},
27};
28use thiserror::Error;
29
30#[derive(Clone, Debug)]
32pub struct TxReport {
33 pub db_before: Db,
35 pub db_after: Db,
37 pub tx: PreparedTx,
39 pub tx_instant: i64,
41}
42
43#[derive(Debug, Error)]
45pub enum TransactError {
46 #[error(transparent)]
48 Tx(#[from] TxError),
49 #[error(transparent)]
51 Log(#[from] LogError),
52 #[error(transparent)]
54 Store(#[from] StoreError),
55 #[error("system clock is before Unix epoch")]
57 Clock,
58 #[error("index task failed: {0}")]
60 IndexTask(String),
61 #[error("an asynchronous transaction is already in progress")]
63 AsyncTransactionPending,
64 #[error("deposed: database root is owned by lease version {published}")]
66 Deposed {
67 published: u64,
69 },
70}
71
72fn seal_tx_instant(
85 datoms: &mut Vec<corium_core::Datom>,
86 t: u64,
87 now_ms: i64,
88 last_instant: i64,
89) -> Result<i64, TxError> {
90 match corium_db::bootstrap::asserted_instant(t, datoms) {
91 Some(supplied) if supplied <= last_instant => Err(TxError::TxInstantNotMonotonic {
92 supplied,
93 last: last_instant,
94 }),
95 Some(supplied) => Ok(supplied),
96 None => {
97 let instant = now_ms.max(last_instant.saturating_add(1));
98 datoms.push(corium_db::bootstrap::tx_instant_datom(t, instant));
99 Ok(instant)
100 }
101 }
102}
103
104fn effective_tx_instant(record: &TxRecord) -> i64 {
105 corium_db::bootstrap::asserted_instant(record.t, &record.datoms).unwrap_or(record.tx_instant)
106}
107
108struct State {
109 db: Db,
110 next_user: u64,
111 last_instant: i64,
112 subscribers: Vec<mpsc::Sender<TxReport>>,
113 async_pending: bool,
114}
115
116struct AsyncPending<'a> {
117 state: &'a Mutex<State>,
118 active: bool,
119}
120
121impl Drop for AsyncPending<'_> {
122 fn drop(&mut self) {
123 if self.active {
124 self.state
125 .lock()
126 .unwrap_or_else(std::sync::PoisonError::into_inner)
127 .async_pending = false;
128 }
129 }
130}
131
132fn next_user_id<'a>(datoms: impl Iterator<Item = &'a corium_core::Datom>, floor: u64) -> u64 {
135 datoms
136 .filter(|d| d.e.partition() == Partition::User as u32)
137 .map(|d| d.e.sequence() + 1)
138 .fold(floor, u64::max)
139}
140
141pub struct Prepared {
145 pub record: TxRecord,
147 db_before: Db,
148 db_after: Db,
149 tx: PreparedTx,
150 tx_instant: i64,
151}
152
153pub struct BatchCursor {
162 db: Db,
163 next_user: u64,
164 last_instant: i64,
165}
166
167impl BatchCursor {
168 #[must_use]
171 pub fn db(&self) -> &Db {
172 &self.db
173 }
174
175 pub fn prepare(
184 &mut self,
185 items: impl IntoIterator<Item = TxItem>,
186 now_ms: i64,
187 ) -> Result<Prepared, TxError> {
188 let before = self.db.clone();
189 let t = before.basis_t() + 1;
190 let tx_id = EntityId::new(Partition::Tx as u32, t);
191 let mut prepared = prepare(&before, items, tx_id, self.next_user)?;
192 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, now_ms, self.last_instant)?;
193 let after = before
194 .clone()
195 .with_transaction_at(t, tx_instant, &prepared.datoms);
196 self.next_user = prepared
197 .tempids
198 .values()
199 .filter(|e| e.partition() == Partition::User as u32)
200 .map(|e| e.sequence() + 1)
201 .max()
202 .unwrap_or(self.next_user)
203 .max(self.next_user);
204 self.last_instant = tx_instant;
205 self.db = after.clone();
206 Ok(Prepared {
207 record: TxRecord {
208 t,
209 tx_instant,
210 datoms: prepared.datoms.clone(),
211 },
212 db_before: before,
213 db_after: after,
214 tx: prepared,
215 tx_instant,
216 })
217 }
218}
219
220pub struct EmbeddedTransactor {
222 log: Arc<dyn TransactionLog>,
223 state: Mutex<State>,
224 async_commit: tokio::sync::Mutex<()>,
225}
226impl EmbeddedTransactor {
227 pub fn recover(schema: Schema, log: Arc<dyn TransactionLog>) -> Result<Self, TransactError> {
232 Self::recover_from(Db::new(schema), log)
233 }
234
235 pub fn recover_from(base: Db, log: Arc<dyn TransactionLog>) -> Result<Self, TransactError> {
241 let mut db = base;
242 let mut last_instant = i64::MIN;
243 for record in log.replay()? {
244 let tx_instant = effective_tx_instant(&record);
245 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
246 last_instant = last_instant.max(tx_instant);
247 }
248 let next_user = next_user_id(db.recorded_datoms(), FIRST_USER_ID);
252 Ok(Self {
253 log,
254 state: Mutex::new(State {
255 db,
256 next_user,
257 last_instant,
258 subscribers: Vec::new(),
259 async_pending: false,
260 }),
261 async_commit: tokio::sync::Mutex::new(()),
262 })
263 }
264
265 pub async fn recover_from_async(
270 base: Db,
271 log: Arc<dyn TransactionLog>,
272 ) -> Result<Self, TransactError> {
273 let records = log.replay_async().await?;
274 Ok(Self::recover_from_records(base, log, records))
275 }
276
277 fn recover_from_records(
278 mut db: Db,
279 log: Arc<dyn TransactionLog>,
280 records: Vec<TxRecord>,
281 ) -> Self {
282 let mut last_instant = i64::MIN;
283 for record in records {
284 let tx_instant = effective_tx_instant(&record);
285 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
286 last_instant = last_instant.max(tx_instant);
287 }
288 let next_user = next_user_id(db.recorded_datoms(), FIRST_USER_ID);
289 Self {
290 log,
291 state: Mutex::new(State {
292 db,
293 next_user,
294 last_instant,
295 subscribers: Vec::new(),
296 async_pending: false,
297 }),
298 async_commit: tokio::sync::Mutex::new(()),
299 }
300 }
301
302 pub fn recover_from_snapshot(
327 snapshot: Db,
328 next_entity_id: u64,
329 last_tx_instant: i64,
330 log: Arc<dyn TransactionLog>,
331 ) -> Result<Self, TransactError> {
332 let mut db = snapshot;
333 let index_basis = db.basis_t();
334 let mut last_instant = last_tx_instant;
335 let mut next_user = next_entity_id.max(FIRST_USER_ID);
338 for record in log.tx_range(index_basis + 1, None)? {
339 let tx_instant = effective_tx_instant(&record);
340 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
341 last_instant = last_instant.max(tx_instant);
342 next_user = next_user.max(next_user_id(record.datoms.iter(), next_user));
343 }
344 Ok(Self {
345 log,
346 state: Mutex::new(State {
347 db,
348 next_user,
349 last_instant,
350 subscribers: Vec::new(),
351 async_pending: false,
352 }),
353 async_commit: tokio::sync::Mutex::new(()),
354 })
355 }
356
357 pub async fn recover_from_snapshot_async(
363 snapshot: Db,
364 next_entity_id: u64,
365 last_tx_instant: i64,
366 log: Arc<dyn TransactionLog>,
367 ) -> Result<Self, TransactError> {
368 let index_basis = snapshot.basis_t();
369 let records = log.tx_range_async(index_basis + 1, None).await?;
370 let mut db = snapshot;
371 let mut last_instant = last_tx_instant;
372 let mut next_user = next_entity_id.max(FIRST_USER_ID);
373 for record in records {
374 let tx_instant = effective_tx_instant(&record);
375 db = db.with_transaction_at(record.t, record.tx_instant, &record.datoms);
376 last_instant = last_instant.max(tx_instant);
377 next_user = next_user.max(next_user_id(record.datoms.iter(), next_user));
378 }
379 Ok(Self {
380 log,
381 state: Mutex::new(State {
382 db,
383 next_user,
384 last_instant,
385 subscribers: Vec::new(),
386 async_pending: false,
387 }),
388 async_commit: tokio::sync::Mutex::new(()),
389 })
390 }
391
392 fn recovery_snapshot(&self) -> (Db, u64, i64) {
396 let state = self
397 .state
398 .lock()
399 .unwrap_or_else(std::sync::PoisonError::into_inner);
400 (state.db.clone(), state.next_user, state.last_instant)
401 }
402 #[must_use]
404 pub fn db(&self) -> Db {
405 self.state
406 .lock()
407 .unwrap_or_else(std::sync::PoisonError::into_inner)
408 .db
409 .clone()
410 }
411 pub fn subscribe(&self) -> mpsc::Receiver<TxReport> {
413 let (tx, rx) = mpsc::channel();
414 self.state
415 .lock()
416 .unwrap_or_else(std::sync::PoisonError::into_inner)
417 .subscribers
418 .push(tx);
419 rx
420 }
421 pub fn transact(
427 &self,
428 items: impl IntoIterator<Item = TxItem>,
429 ) -> Result<TxReport, TransactError> {
430 let mut state = self
431 .state
432 .lock()
433 .unwrap_or_else(std::sync::PoisonError::into_inner);
434 if state.async_pending {
435 return Err(TransactError::AsyncTransactionPending);
436 }
437 let before = state.db.clone();
438 let t = before.basis_t() + 1;
439 let tx_id = EntityId::new(Partition::Tx as u32, t);
440 let mut prepared = prepare(&before, items, tx_id, state.next_user)?;
441 let millis = i64::try_from(
442 SystemTime::now()
443 .duration_since(UNIX_EPOCH)
444 .map_err(|_| TransactError::Clock)?
445 .as_millis(),
446 )
447 .unwrap_or(i64::MAX);
448 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, millis, state.last_instant)?;
449 self.log.append(&TxRecord {
450 t,
451 tx_instant,
452 datoms: prepared.datoms.clone(),
453 })?;
454 state.db = before.with_transaction_at(t, tx_instant, &prepared.datoms);
455 state.last_instant = tx_instant;
456 state.next_user = prepared
457 .tempids
458 .values()
459 .filter(|e| e.partition() == Partition::User as u32)
460 .map(|e| e.sequence() + 1)
461 .max()
462 .unwrap_or(state.next_user)
463 .max(state.next_user);
464 let report = TxReport {
465 db_before: before,
466 db_after: state.db.clone(),
467 tx: prepared,
468 tx_instant,
469 };
470 state
471 .subscribers
472 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
473 Ok(report)
474 }
475
476 pub async fn transact_async(
485 &self,
486 items: impl IntoIterator<Item = TxItem>,
487 ) -> Result<TxReport, TransactError> {
488 let _commit = self.async_commit.lock().await;
489 let (before, prepared, t, tx_instant) = {
490 let mut state = self
491 .state
492 .lock()
493 .unwrap_or_else(std::sync::PoisonError::into_inner);
494 if state.async_pending {
495 return Err(TransactError::AsyncTransactionPending);
496 }
497 let before = state.db.clone();
498 let t = before.basis_t() + 1;
499 let tx_id = EntityId::new(Partition::Tx as u32, t);
500 let mut prepared = prepare(&before, items, tx_id, state.next_user)?;
501 let millis = i64::try_from(
502 SystemTime::now()
503 .duration_since(UNIX_EPOCH)
504 .map_err(|_| TransactError::Clock)?
505 .as_millis(),
506 )
507 .unwrap_or(i64::MAX);
508 let tx_instant = seal_tx_instant(&mut prepared.datoms, t, millis, state.last_instant)?;
509 state.async_pending = true;
510 (before, prepared, t, tx_instant)
511 };
512 let record = TxRecord {
513 t,
514 tx_instant,
515 datoms: prepared.datoms.clone(),
516 };
517 let mut pending = AsyncPending {
518 state: &self.state,
519 active: true,
520 };
521 self.log.append_async(&record).await?;
522 let mut state = self
523 .state
524 .lock()
525 .unwrap_or_else(std::sync::PoisonError::into_inner);
526 debug_assert_eq!(state.db.basis_t() + 1, t);
527 state.db = state
531 .db
532 .clone()
533 .with_transaction_at(t, tx_instant, &prepared.datoms);
534 state.last_instant = tx_instant;
535 state.next_user = prepared
536 .tempids
537 .values()
538 .filter(|e| e.partition() == Partition::User as u32)
539 .map(|e| e.sequence() + 1)
540 .max()
541 .unwrap_or(state.next_user)
542 .max(state.next_user);
543 state.async_pending = false;
544 pending.active = false;
545 let report = TxReport {
546 db_before: before,
547 db_after: state.db.clone(),
548 tx: prepared,
549 tx_instant,
550 };
551 state
552 .subscribers
553 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
554 Ok(report)
555 }
556 #[must_use]
560 pub fn batch_cursor(&self) -> BatchCursor {
561 let state = self
562 .state
563 .lock()
564 .unwrap_or_else(std::sync::PoisonError::into_inner);
565 BatchCursor {
566 db: state.db.clone(),
567 next_user: state.next_user,
568 last_instant: state.last_instant,
569 }
570 }
571
572 pub fn install_batch(&self, cursor: BatchCursor, prepared: Vec<Prepared>) -> Vec<TxReport> {
580 let mut state = self
581 .state
582 .lock()
583 .unwrap_or_else(std::sync::PoisonError::into_inner);
584 state.db = cursor.db;
585 state.next_user = state.next_user.max(cursor.next_user);
586 state.last_instant = state.last_instant.max(cursor.last_instant);
587 let reports: Vec<TxReport> = prepared
588 .into_iter()
589 .map(|p| TxReport {
590 db_before: p.db_before,
591 db_after: p.db_after,
592 tx: p.tx,
593 tx_instant: p.tx_instant,
594 })
595 .collect();
596 for report in &reports {
597 state
598 .subscribers
599 .retain(|subscriber| subscriber.send(report.clone()).is_ok());
600 }
601 reports
602 }
603
604 pub fn update_naming(&self, idents: Idents, interner: KeywordInterner) {
607 let mut state = self
608 .state
609 .lock()
610 .unwrap_or_else(std::sync::PoisonError::into_inner);
611 state.db = state.db.clone().with_naming(idents, interner);
612 }
613
614 pub async fn publish_indexes(
635 &self,
636 store: &(impl BlobStore + RootStore),
637 root_name: &str,
638 lease_version: u64,
639 ) -> Result<DbRoot, TransactError> {
640 let (snapshot, next_entity_id, last_tx_instant) = self.recovery_snapshot();
641 let datoms = snapshot.datoms();
642 let chunked = tokio::task::spawn_blocking(move || {
643 [
644 IndexOrder::Eavt,
645 IndexOrder::Aevt,
646 IndexOrder::Avet,
647 IndexOrder::Vaet,
648 ]
649 .into_iter()
650 .map(|order| {
651 let segment = Segment::build(order, datoms.clone());
652 corium_store::chunk_segment_keys(segment.entries().map(|(key, _)| key.as_slice()))
653 })
654 .collect::<Vec<_>>()
655 })
656 .await
657 .map_err(|error| TransactError::IndexTask(error.to_string()))?;
658 let mut ids = Vec::new();
659 for chunks in chunked {
660 let mut children = Vec::new();
661 for chunk in chunks {
662 children.push(store.put_if_absent(&chunk).await?);
663 }
664 let manifest = corium_store::encode_index_manifest(&children);
665 ids.push(store.put_if_absent(&manifest).await?);
666 }
667 let root = DbRoot {
668 format_version: corium_store::FORMAT_VERSION,
669 lease_version,
670 owner: String::new(),
671 lease_expires_unix_ms: 0,
672 owner_endpoint: String::new(),
673 index_basis_t: snapshot.basis_t(),
674 roots: Some([
675 ids[0].clone(),
676 ids[1].clone(),
677 ids[2].clone(),
678 ids[3].clone(),
679 ]),
680 next_entity_id,
682 last_tx_instant,
683 };
684 publish_root(store, root_name, &root).await?;
685 Ok(root)
686 }
687}
688
689pub async fn publish_root(
696 store: &dyn RootStore,
697 root_name: &str,
698 root: &DbRoot,
699) -> Result<(), TransactError> {
700 loop {
701 let previous = store.get_root(root_name).await?;
702 let stored = previous.as_deref().and_then(DbRoot::decode);
703 let mut next = root.clone();
704 if let Some(stored) = stored {
705 if stored.lease_version > root.lease_version {
706 return Err(TransactError::Deposed {
707 published: stored.lease_version,
708 });
709 }
710 if stored.lease_version == root.lease_version
711 && stored.index_basis_t >= root.index_basis_t
712 {
713 return Ok(());
714 }
715 if stored.lease_version == root.lease_version {
718 next.owner = stored.owner;
719 next.lease_expires_unix_ms = stored.lease_expires_unix_ms;
720 next.owner_endpoint = stored.owner_endpoint;
721 }
722 }
723 match store
724 .cas_root(root_name, previous.as_deref(), &next.encode())
725 .await
726 {
727 Ok(()) => return Ok(()),
728 Err(StoreError::CasFailed { .. }) => {}
729 Err(error) => return Err(error.into()),
730 }
731 }
732}
733
734pub use corium_store::{DbRoot, db_root_name};