1mod admission;
10mod codec;
11mod commit_cache;
12mod connection_functions;
13mod identifiers;
14mod key_value;
15#[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
16pub(crate) mod leases;
17pub mod native;
18mod read;
19pub(crate) mod receipts;
20mod reclamation;
21mod resources;
22pub(crate) mod restore;
23mod retention;
24mod runs;
25mod schema;
26mod sequence_durability;
27mod serializable;
28mod synchronization;
29#[cfg(test)]
30mod tests;
31mod tombstones;
32mod version_metadata;
33mod write;
34
35pub(crate) use schema::WritePermit;
36pub use serializable::SQLiteSerializableAdmission;
37
38use std::sync::Arc;
39
40use rusqlite::Connection;
41use uqa_storage::mvcc::{
42 CommitReceipt, CommitResult, CommitStatus, CommittedRecordSnapshot, DatabaseId,
43 PreparedRecordCommit, StorageTransactionId, VersionError, VersionResult, VersionedPersistence,
44};
45use uqa_storage::read_control::StorageReadControl;
46
47use crate::{ManagedConnection, SQLiteError};
48
49#[derive(Debug, thiserror::Error)]
50enum Error {
51 #[error(transparent)]
52 SQLite(#[from] rusqlite::Error),
53 #[error(transparent)]
54 Version(#[from] VersionError),
55}
56
57impl Error {
58 fn into_version(self) -> VersionError {
59 match self {
60 Self::SQLite(error) => sqlite_error(error),
61 Self::Version(error) => error,
62 }
63 }
64}
65
66impl From<SQLiteError> for Error {
67 fn from(error: SQLiteError) -> Self {
68 match error {
69 SQLiteError::Memory(error) => Self::Version(VersionError::Memory(error)),
70 SQLiteError::Cancelled(error) => Self::Version(VersionError::Cancelled(error)),
71 SQLiteError::SQLite(error) => Self::SQLite(error),
72 error => Self::Version(VersionError::Storage(error.into())),
73 }
74 }
75}
76
77type PhysicalResult<T> = std::result::Result<T, Error>;
78
79fn sqlite_error(error: rusqlite::Error) -> VersionError {
80 VersionError::Storage(SQLiteError::SQLite(error).into())
81}
82
83#[derive(Clone)]
87pub struct SQLiteRecordStore {
88 connection: ManagedConnection,
89 identity: DatabaseId,
90 native: Option<native::NativeRecordNamespace>,
91 sequence_publication: Option<uqa_storage::mvcc::SequencePublicationLease>,
92 snapshots: Arc<uqa_storage::mvcc::SnapshotRegistry>,
93 managed_allocations: receipts::ManagedAllocations,
94 data_commit: Arc<parking_lot::Mutex<Option<CommitReceipt>>>,
96 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
97 receipt_leases: receipts::ReceiptLeaseFile,
98}
99
100#[cfg(test)]
101thread_local! {
102 static READ_VALIDATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
104 static RECORD_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
106}
107
108impl SQLiteRecordStore {
109 pub(crate) fn has_native_mapping(connection: &Connection) -> VersionResult<bool> {
110 native::present(connection).map_err(Error::into_version)
111 }
112
113 pub(crate) fn initialize_native_in(
115 connection: &ManagedConnection,
116 transaction: &Connection,
117 control: &StorageReadControl,
118 ) -> VersionResult<Self> {
119 let native::NativeMapping {
120 identity,
121 namespace,
122 } = native::initialize_in(transaction, control).map_err(Error::into_version)?;
123 Ok(Self {
124 managed_allocations: receipts::ManagedAllocations::default(),
125 data_commit: Arc::default(),
126 sequence_publication: None,
127 snapshots: retention::registry(connection, identity)?,
128 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
129 receipt_leases: receipts::ReceiptLeaseFile::default(),
130 connection: connection.record_connection(),
131 identity,
132 native: Some(namespace),
133 })
134 }
135
136 pub fn new(connection: &ManagedConnection) -> VersionResult<Self> {
137 if connection.in_transaction() {
138 return Err(VersionError::Storage(
139 SQLiteError::TransactionAlreadyActive.into(),
140 ));
141 }
142 let control = StorageReadControl::with_limit(
143 uqa_storage::mvcc::VersionedSessionOptions::default().retained_bytes,
144 );
145 let _admission = connection
146 .admit_record_initialization(&control)
147 .map_err(|error| VersionError::Storage(error.into()))?;
148 let connection = connection.record_connection();
149 let identity = connection
150 .with(|connection| Ok(schema::initialize(connection)))
151 .map_err(|error| VersionError::Storage(error.into()))?
152 .map_err(Error::into_version)?;
153 Ok(Self {
154 managed_allocations: receipts::ManagedAllocations::default(),
155 data_commit: Arc::default(),
156 sequence_publication: None,
157 snapshots: retention::registry(&connection, identity)?,
158 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
159 receipt_leases: receipts::ReceiptLeaseFile::default(),
160 connection,
161 identity,
162 native: None,
163 })
164 }
165
166 pub fn for_native(
168 connection: &ManagedConnection,
169 control: &StorageReadControl,
170 ) -> VersionResult<Self> {
171 if connection.in_transaction() {
172 return Err(VersionError::Storage(
173 SQLiteError::TransactionAlreadyActive.into(),
174 ));
175 }
176 let _admission = connection
177 .admit_record_initialization(control)
178 .map_err(|error| VersionError::Storage(error.into()))?;
179 let connection = connection.record_connection();
180 let native::NativeMapping {
181 identity,
182 namespace,
183 } = connection
184 .with(|connection| Ok(native::initialize(connection, control)))
185 .map_err(|error| VersionError::Storage(error.into()))?
186 .map_err(Error::into_version)?;
187 Ok(Self {
188 managed_allocations: receipts::ManagedAllocations::default(),
189 data_commit: Arc::default(),
190 sequence_publication: None,
191 snapshots: retention::registry(&connection, identity)?,
192 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
193 receipt_leases: receipts::ReceiptLeaseFile::default(),
194 connection,
195 identity,
196 native: Some(namespace),
197 })
198 }
199
200 pub(crate) fn for_key_value(
201 connection: &ManagedConnection,
202 control: &StorageReadControl,
203 ) -> VersionResult<Self> {
204 let _admission = connection
205 .admit_record_initialization(control)
206 .map_err(|error| VersionError::Storage(error.into()))?;
207 let connection = connection.record_connection();
208 let identity = connection
209 .with(|connection| Ok(key_value::initialize(connection, control)))
210 .map_err(|error| VersionError::Storage(error.into()))?
211 .map_err(Error::into_version)?;
212 Ok(Self {
213 managed_allocations: receipts::ManagedAllocations::default(),
214 data_commit: Arc::default(),
215 sequence_publication: None,
216 snapshots: retention::registry(&connection, identity)?,
217 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
218 receipt_leases: receipts::ReceiptLeaseFile::default(),
219 connection,
220 identity,
221 native: None,
222 })
223 }
224
225 #[must_use]
227 pub fn native_namespace(&self) -> Option<DatabaseId> {
228 self.native.map(|namespace| namespace.0)
229 }
230
231 fn with<T>(
232 &self,
233 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
234 ) -> VersionResult<T> {
235 self.connection
236 .with(|connection| Ok(operation(connection)))
237 .map_err(|error| VersionError::Storage(error.into()))?
238 .map_err(Error::into_version)
239 }
240
241 fn read<T>(
243 &self,
244 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
245 ) -> VersionResult<T> {
246 #[cfg(test)]
247 RECORD_READS.with(|count| count.set(count.get() + 1));
248 self.connection
249 .with_record_read(|connection, validated| {
250 Ok((|| {
251 let transaction = connection.unchecked_transaction()?;
252 let state = crate::connection::ValidatedRead {
253 store: (
254 self.identity.as_bytes(),
255 self.native.map(|namespace| namespace.0.as_bytes()),
256 ),
257 data_version: transaction
258 .prepare_cached("PRAGMA data_version")?
259 .query_row([], |row| row.get(0))?,
260 changes: connection.total_changes(),
261 };
262 if validated.get() != Some(state) {
263 native::check_mapping(&transaction, self.native)?;
264 codec::header(&transaction, self.identity)?;
265 validated.set(Some(state));
266 #[cfg(test)]
267 READ_VALIDATIONS.with(|count| count.set(count.get() + 1));
268 }
269 let result = operation(&transaction)?;
270 transaction.commit()?;
271 Ok(result)
272 })())
273 })
274 .map_err(|error| VersionError::Storage(error.into()))?
275 .map_err(Error::into_version)
276 }
277
278 fn with_write<T>(
279 &self,
280 control: &StorageReadControl,
281 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
282 ) -> VersionResult<T> {
283 let mut operation = Some(operation);
284 loop {
285 control.cancellation().check()?;
286 let result = self
287 .connection
288 .with_record_connection(control, |connection| {
289 let _timeout = admission::BusyTimeout::new(connection)?;
290 control.cancellation().check()?;
291 Ok(operation.take().expect("record write runs once")(
292 connection,
293 ))
294 });
295 match result {
296 Err(SQLiteError::SQLite(error))
297 if operation.is_some() && admission::is_busy(&error) =>
298 {
299 admission::wait(control).map_err(Error::into_version)?;
301 }
302 result => {
303 return result
304 .map_err(|error| Error::from(error).into_version())?
305 .map_err(Error::into_version);
306 }
307 }
308 }
309 }
310
311 fn check_transaction(&self, transaction: StorageTransactionId) -> VersionResult<()> {
312 if transaction.database() != self.identity {
313 return Err(VersionError::WrongDatabase);
314 }
315 Ok(())
316 }
317}
318
319impl VersionedPersistence for SQLiteRecordStore {
320 fn sequence_value_persistence(
321 &self,
322 owner: uqa_storage::mvcc::SequencePublicationLease,
323 ) -> Option<Arc<dyn VersionedPersistence>> {
324 self.native?;
325 let mut records = self.clone();
326 records.sequence_publication = Some(owner);
327 Some(Arc::new(records))
328 }
329
330 fn synchronize_sequence_values(&self, control: &StorageReadControl) -> VersionResult<()> {
331 self.with_write(control, |connection| {
332 sequence_durability::synchronize(connection, self.identity, self.native, control)
333 })
334 }
335
336 fn commit_preserves_catalog_definitions(&self, receipt: CommitReceipt) -> bool {
337 *self.data_commit.lock() == Some(receipt)
338 }
339
340 fn diskann_population_record_layout(
341 &self,
342 ) -> Option<&dyn uqa_storage::mvcc::DiskANNPopulationRecordLayout> {
343 Some(match &self.native {
344 Some(namespace) => namespace,
345 None => &uqa_storage::key_value::KeyValueDiskANNPopulationRecords,
346 })
347 }
348
349 fn resource_leases(&self) -> Option<&dyn uqa_storage::mvcc::ResourceLeaseProvider> {
350 Some(self)
351 }
352
353 fn serializable_coordinator(&self) -> Option<&dyn uqa_storage::mvcc::SerializableCoordinator> {
354 Some(self)
355 }
356
357 fn identifier_watermark(
358 &self,
359 namespace: &[u8],
360 control: &StorageReadControl,
361 ) -> VersionResult<Option<u64>> {
362 control.cancellation().check()?;
363 self.with(|connection| {
364 identifiers::read(connection, self.identity, self.native, namespace, control)
365 })
366 }
367
368 fn allocate_identifiers(
369 &self,
370 namespace: &[u8],
371 request: uqa_storage::mvcc::IdentifierRequest,
372 control: &StorageReadControl,
373 ) -> VersionResult<uqa_storage::mvcc::IdentifierAllocation> {
374 control.cancellation().check()?;
375 self.with_write(control, |connection| {
376 identifiers::allocate(
377 connection,
378 self.identity,
379 self.native,
380 namespace,
381 request,
382 control,
383 )
384 })
385 }
386
387 fn database_id(&self) -> DatabaseId {
388 self.identity
389 }
390 fn private_revision_scope(&self) -> Option<uqa_storage::mvcc::PrivateRevisionScope> {
391 self.native
392 .is_some()
393 .then_some(|key| Ok(native::NativeRecordIdentity::revision_scope(key)))
394 }
395 fn graph_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::GraphRecordLayout> {
396 match self.native.as_ref() {
397 Some(namespace) => Some(namespace),
398 None => Some(&uqa_storage::key_value::KeyValueGraphRecords),
399 }
400 }
401 fn occurrence_record_layout(&self) -> &dyn uqa_storage::mvcc::OccurrenceRecordLayout {
402 if self.native.is_some() {
403 &crate::inverted_index::NativeOccurrenceRecords
404 } else {
405 &uqa_storage::key_value::KeyValueOccurrenceRecords
406 }
407 }
408 fn notification_record_layout(&self) -> &dyn uqa_storage::mvcc::NotificationRecordLayout {
409 match self.native.as_ref() {
410 Some(namespace) => namespace,
411 None => &uqa_storage::key_value::KeyValueNotificationRecords,
412 }
413 }
414
415 fn maintenance_record_layout(&self) -> &dyn uqa_storage::mvcc::MaintenanceRecordLayout {
416 if self.native.is_some() {
417 &native::NativeMaintenanceRecords
418 } else {
419 &uqa_storage::key_value::KeyValueMaintenanceRecords
420 }
421 }
422 fn ivf_record_layout(&self) -> &dyn uqa_storage::mvcc::IVFRecordLayout {
423 if self.native.is_some() {
424 &crate::vector_index::NativeIVFRecords
425 } else {
426 &uqa_storage::key_value::KeyValueIVFRecords
427 }
428 }
429 fn hnsw_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::HNSWRecordLayout> {
430 if self.native.is_some() {
431 Some(&crate::vector_index::NativeHNSWRecords)
432 } else {
433 Some(&uqa_storage::key_value::KeyValueHNSWRecords)
434 }
435 }
436 fn allocate_transaction(
437 &self,
438 control: &StorageReadControl,
439 ) -> VersionResult<StorageTransactionId> {
440 control.cancellation().check()?;
441 self.with_write(control, |connection| {
442 write::allocate(connection, self.identity, self.native, control)
443 })
444 }
445
446 fn allocate_managed_transaction(
447 &self,
448 control: &StorageReadControl,
449 ) -> VersionResult<uqa_storage::mvcc::RetainedTransactionAllocation> {
450 self.allocate_receipt_owner(control)
451 }
452 fn snapshot(
453 &self,
454 control: &StorageReadControl,
455 ) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
456 control.cancellation().check()?;
457 let monitor = self.commit_monitor_version()?;
459 let mut reclamation_epoch = 0;
460 let lease = self.snapshots.capture(control, || {
461 self.with(|connection| {
462 let read = connection.unchecked_transaction()?;
463 native::check_mapping(&read, self.native)?;
464 let sequence = codec::header(&read, self.identity)?.sequence;
465 reclamation_epoch = tombstones::epoch(&read)?;
466 read.commit()?;
467 Ok(sequence)
468 })
469 })?;
470 uqa_storage::mvcc::retain_record_snapshot(
471 read::Snapshot {
472 store: self.clone(),
473 sequence: lease.sequence(),
474 reclamation_epoch,
475 monitor: monitor.map(std::sync::atomic::AtomicU64::new),
476 table_owners: read::table_owners::TableOwners::default(),
477 row_presence: read::row_presence::RowPresence::default(),
478 _lease: lease,
479 },
480 control,
481 )
482 }
483 fn commit_monitor_version(&self) -> VersionResult<Option<u64>> {
484 self.connection
485 .commit_monitor_version()
486 .map_err(|error| VersionError::Storage(error.into()))
487 }
488 fn reclaim_versions(&self, control: &StorageReadControl) -> VersionResult<u64> {
489 self.snapshots.reclaim(control, |oldest| {
490 self.with_write(control, |connection| {
491 reclamation::reclaim(connection, self.identity, self.native, oldest, control)
492 })
493 })
494 }
495
496 fn reclaim_tombstones(
497 &self,
498 request: &uqa_storage::mvcc::TombstoneReclamationRequest<'_>,
499 control: &StorageReadControl,
500 ) -> VersionResult<uqa_storage::mvcc::TombstoneReclamationStep> {
501 request.validate(control)?;
502 self.snapshots.reclaim(control, |oldest| {
503 if oldest.is_some() {
504 return Ok(uqa_storage::mvcc::TombstoneReclamationStep::Retained);
505 }
506 self.with_write(control, |connection| {
507 tombstones::reclaim(connection, self.identity, self.native, request, control)
508 })
509 })
510 }
511
512 fn reclaim_diskann_tombstones(&self, control: &StorageReadControl) -> VersionResult<()> {
513 if self.native.is_none() {
514 return uqa_storage::mvcc::reclaim_key_value_diskann_tombstones(self, control);
515 }
516 for family in [
517 native::NativeRecordFamily::DiskANNRecords,
518 native::NativeRecordFamily::VectorOrigins,
519 native::NativeRecordFamily::VectorChanges,
520 native::NativeRecordFamily::VectorPopulations,
521 native::NativeRecordFamily::VectorPopulationWitnesses,
522 ] {
523 let prefix = native::NativeRecordIdentity::family_prefix(family, control)?;
524 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
525 }
526 let prefix = self.vector_field_guard_layout().prefix(control)?;
527 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
528 Ok(())
529 }
530
531 fn vector_field_guard_layout(&self) -> &dyn uqa_storage::mvcc::VectorFieldGuardLayout {
532 self
533 }
534
535 fn commit(
536 &self,
537 transaction: StorageTransactionId,
538 prepared: &PreparedRecordCommit,
539 control: &StorageReadControl,
540 ) -> CommitResult {
541 self.check_transaction(transaction)?;
542 control.cancellation().check().map_err(VersionError::from)?;
543 let _bindings = write::reserve_bindings(prepared, control)?;
544 let publish = |deferred| {
545 self.with_write(control, |connection| {
546 let _cache = commit_cache::CommitCache::grow(connection, prepared, control)?;
547 Ok(write::commit(
548 connection,
549 transaction,
550 prepared,
551 self.native,
552 deferred,
553 control,
554 |receipt| *self.data_commit.lock() = Some(receipt),
555 ))
556 })?
557 };
558 match &self.sequence_publication {
559 Some(owner) => owner.with_publication(publish),
560 None => publish(false),
561 }
562 }
563 fn commit_status(
564 &self,
565 transaction: StorageTransactionId,
566 control: &StorageReadControl,
567 ) -> VersionResult<CommitStatus> {
568 self.check_transaction(transaction)?;
569 control.cancellation().check()?;
570 self.with(|connection| {
571 let read = connection.unchecked_transaction()?;
572 native::check_mapping(&read, self.native)?;
573 codec::header(&read, self.identity)?;
574 let status = codec::status(&read, transaction)?;
575 control.cancellation().check().map_err(VersionError::from)?;
576 read.commit()?;
577 Ok(status)
578 })
579 }
580
581 fn acknowledge_transaction(
582 &self,
583 acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
584 control: &StorageReadControl,
585 ) -> VersionResult<()> {
586 self.check_transaction(acknowledgement.transaction())?;
587 self.with_write(control, |connection| {
588 receipts::acknowledge(connection, self.native, acknowledgement, control)
589 })
590 }
591
592 fn reclaim_transaction_receipts(&self, control: &StorageReadControl) -> VersionResult<u64> {
593 self.reclaim_receipts(control)
594 }
595
596 fn acknowledge_retained_transaction(
597 &self,
598 owner: &uqa_storage::mvcc::RetainedTransactionAllocation,
599 acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
600 control: &StorageReadControl,
601 ) -> VersionResult<()> {
602 self.check_transaction(acknowledgement.transaction())?;
603 if owner.transaction() != acknowledgement.transaction() {
604 return Err(VersionError::InvalidEncoding(
605 "receipt acknowledgement owner mismatch",
606 ));
607 }
608 self.with_write(control, |connection| {
609 if owner.has_resolution_lease()
610 && receipts::validate_retained(connection, self.native, acknowledgement, control)?
611 {
612 return Ok(());
613 }
614 receipts::acknowledge(connection, self.native, acknowledgement, control)
615 })
616 }
617 fn abort(
618 &self,
619 transaction: StorageTransactionId,
620 control: &StorageReadControl,
621 ) -> VersionResult<CommitStatus> {
622 self.check_transaction(transaction)?;
623 control.cancellation().check()?;
624 self.with(|connection| write::abort(connection, transaction, self.native, control))
625 }
626}