1mod admission;
10mod codec;
11mod commit_cache;
12mod connection_functions;
13mod identifiers;
14mod key_value;
15#[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
16pub(crate) mod leases;
17pub mod native;
18mod read;
19pub(crate) mod receipts;
20mod reclamation;
21mod resources;
22pub(crate) mod restore;
23mod retention;
24mod runs;
25mod schema;
26mod serializable;
27mod synchronization;
28#[cfg(test)]
29mod tests;
30mod tombstones;
31mod version_metadata;
32mod write;
33
34pub(crate) use schema::WritePermit;
35pub use serializable::SQLiteSerializableAdmission;
36
37use std::sync::Arc;
38
39use rusqlite::Connection;
40use uqa_storage::mvcc::{
41 CommitResult, CommitStatus, CommittedRecordSnapshot, DatabaseId, PreparedRecordCommit,
42 StorageTransactionId, VersionError, VersionResult, VersionedPersistence,
43};
44use uqa_storage::read_control::StorageReadControl;
45
46use crate::{ManagedConnection, SQLiteError};
47
48#[derive(Debug, thiserror::Error)]
49enum Error {
50 #[error(transparent)]
51 SQLite(#[from] rusqlite::Error),
52 #[error(transparent)]
53 Version(#[from] VersionError),
54}
55
56impl Error {
57 fn into_version(self) -> VersionError {
58 match self {
59 Self::SQLite(error) => sqlite_error(error),
60 Self::Version(error) => error,
61 }
62 }
63}
64
65impl From<SQLiteError> for Error {
66 fn from(error: SQLiteError) -> Self {
67 match error {
68 SQLiteError::Memory(error) => Self::Version(VersionError::Memory(error)),
69 SQLiteError::Cancelled(error) => Self::Version(VersionError::Cancelled(error)),
70 SQLiteError::SQLite(error) => Self::SQLite(error),
71 error => Self::Version(VersionError::Storage(error.into())),
72 }
73 }
74}
75
76type PhysicalResult<T> = std::result::Result<T, Error>;
77
78fn sqlite_error(error: rusqlite::Error) -> VersionError {
79 VersionError::Storage(SQLiteError::SQLite(error).into())
80}
81
82#[derive(Clone)]
86pub struct SQLiteRecordStore {
87 connection: ManagedConnection,
88 identity: DatabaseId,
89 native: Option<native::NativeRecordNamespace>,
90 snapshots: Arc<uqa_storage::mvcc::SnapshotRegistry>,
91 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
92 receipt_leases: receipts::ReceiptLeaseFile,
93}
94
95#[cfg(test)]
96thread_local! {
97 static READ_VALIDATIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
99 static RECORD_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
101}
102
103impl SQLiteRecordStore {
104 pub(crate) fn has_native_mapping(connection: &Connection) -> VersionResult<bool> {
105 native::present(connection).map_err(Error::into_version)
106 }
107
108 pub(crate) fn initialize_native_in(
110 connection: &ManagedConnection,
111 transaction: &Connection,
112 control: &StorageReadControl,
113 ) -> VersionResult<Self> {
114 let native::NativeMapping {
115 identity,
116 namespace,
117 } = native::initialize_in(transaction, control).map_err(Error::into_version)?;
118 Ok(Self {
119 snapshots: retention::registry(connection, identity)?,
120 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
121 receipt_leases: receipts::ReceiptLeaseFile::default(),
122 connection: connection.record_connection(),
123 identity,
124 native: Some(namespace),
125 })
126 }
127
128 pub fn new(connection: &ManagedConnection) -> VersionResult<Self> {
129 if connection.in_transaction() {
130 return Err(VersionError::Storage(
131 SQLiteError::TransactionAlreadyActive.into(),
132 ));
133 }
134 let connection = connection.record_connection();
135 let identity = connection
136 .with(|connection| Ok(schema::initialize(connection)))
137 .map_err(|error| VersionError::Storage(error.into()))?
138 .map_err(Error::into_version)?;
139 Ok(Self {
140 snapshots: retention::registry(&connection, identity)?,
141 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
142 receipt_leases: receipts::ReceiptLeaseFile::default(),
143 connection,
144 identity,
145 native: None,
146 })
147 }
148
149 pub fn for_native(
151 connection: &ManagedConnection,
152 control: &StorageReadControl,
153 ) -> VersionResult<Self> {
154 if connection.in_transaction() {
155 return Err(VersionError::Storage(
156 SQLiteError::TransactionAlreadyActive.into(),
157 ));
158 }
159 let connection = connection.record_connection();
160 let native::NativeMapping {
161 identity,
162 namespace,
163 } = connection
164 .with(|connection| Ok(native::initialize(connection, control)))
165 .map_err(|error| VersionError::Storage(error.into()))?
166 .map_err(Error::into_version)?;
167 Ok(Self {
168 snapshots: retention::registry(&connection, identity)?,
169 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
170 receipt_leases: receipts::ReceiptLeaseFile::default(),
171 connection,
172 identity,
173 native: Some(namespace),
174 })
175 }
176
177 pub(crate) fn for_key_value(
178 connection: &ManagedConnection,
179 control: &StorageReadControl,
180 ) -> VersionResult<Self> {
181 let connection = connection.record_connection();
182 let identity = connection
183 .with(|connection| Ok(key_value::initialize(connection, control)))
184 .map_err(|error| VersionError::Storage(error.into()))?
185 .map_err(Error::into_version)?;
186 Ok(Self {
187 snapshots: retention::registry(&connection, identity)?,
188 #[cfg(any(windows, all(unix, not(target_os = "emscripten"))))]
189 receipt_leases: receipts::ReceiptLeaseFile::default(),
190 connection,
191 identity,
192 native: None,
193 })
194 }
195
196 #[must_use]
198 pub fn native_namespace(&self) -> Option<DatabaseId> {
199 self.native.map(|namespace| namespace.0)
200 }
201
202 fn with<T>(
203 &self,
204 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
205 ) -> VersionResult<T> {
206 self.connection
207 .with(|connection| Ok(operation(connection)))
208 .map_err(|error| VersionError::Storage(error.into()))?
209 .map_err(Error::into_version)
210 }
211
212 fn read<T>(
214 &self,
215 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
216 ) -> VersionResult<T> {
217 #[cfg(test)]
218 RECORD_READS.with(|count| count.set(count.get() + 1));
219 self.connection
220 .with_record_read(|connection, validated| {
221 Ok((|| {
222 let transaction = connection.unchecked_transaction()?;
223 let state = crate::connection::ValidatedRead {
224 store: (
225 self.identity.as_bytes(),
226 self.native.map(|namespace| namespace.0.as_bytes()),
227 ),
228 data_version: transaction
229 .prepare_cached("PRAGMA data_version")?
230 .query_row([], |row| row.get(0))?,
231 changes: connection.total_changes(),
232 };
233 if validated.get() != Some(state) {
234 native::check_mapping(&transaction, self.native)?;
235 codec::header(&transaction, self.identity)?;
236 validated.set(Some(state));
237 #[cfg(test)]
238 READ_VALIDATIONS.with(|count| count.set(count.get() + 1));
239 }
240 let result = operation(&transaction)?;
241 transaction.commit()?;
242 Ok(result)
243 })())
244 })
245 .map_err(|error| VersionError::Storage(error.into()))?
246 .map_err(Error::into_version)
247 }
248
249 fn with_write<T>(
250 &self,
251 control: &StorageReadControl,
252 operation: impl FnOnce(&Connection) -> PhysicalResult<T>,
253 ) -> VersionResult<T> {
254 let mut operation = Some(operation);
255 loop {
256 control.cancellation().check()?;
257 let result = self
258 .connection
259 .with_record_connection(control, |connection| {
260 let _timeout = admission::BusyTimeout::new(connection)?;
261 control.cancellation().check()?;
262 Ok(operation.take().expect("record write runs once")(
263 connection,
264 ))
265 });
266 match result {
267 Err(SQLiteError::SQLite(error))
268 if operation.is_some() && admission::is_busy(&error) =>
269 {
270 admission::wait(control).map_err(Error::into_version)?;
272 }
273 result => {
274 return result
275 .map_err(|error| Error::from(error).into_version())?
276 .map_err(Error::into_version);
277 }
278 }
279 }
280 }
281
282 fn check_transaction(&self, transaction: StorageTransactionId) -> VersionResult<()> {
283 if transaction.database() != self.identity {
284 return Err(VersionError::WrongDatabase);
285 }
286 Ok(())
287 }
288}
289
290impl VersionedPersistence for SQLiteRecordStore {
291 fn diskann_population_record_layout(
292 &self,
293 ) -> Option<&dyn uqa_storage::mvcc::DiskANNPopulationRecordLayout> {
294 Some(match &self.native {
295 Some(namespace) => namespace,
296 None => &uqa_storage::key_value::KeyValueDiskANNPopulationRecords,
297 })
298 }
299
300 fn resource_leases(&self) -> Option<&dyn uqa_storage::mvcc::ResourceLeaseProvider> {
301 Some(self)
302 }
303
304 fn serializable_coordinator(&self) -> Option<&dyn uqa_storage::mvcc::SerializableCoordinator> {
305 Some(self)
306 }
307
308 fn identifier_watermark(
309 &self,
310 namespace: &[u8],
311 control: &StorageReadControl,
312 ) -> VersionResult<Option<u64>> {
313 control.cancellation().check()?;
314 self.with(|connection| {
315 identifiers::read(connection, self.identity, self.native, namespace, control)
316 })
317 }
318
319 fn allocate_identifiers(
320 &self,
321 namespace: &[u8],
322 request: uqa_storage::mvcc::IdentifierRequest,
323 control: &StorageReadControl,
324 ) -> VersionResult<uqa_storage::mvcc::IdentifierAllocation> {
325 control.cancellation().check()?;
326 self.with_write(control, |connection| {
327 identifiers::allocate(
328 connection,
329 self.identity,
330 self.native,
331 namespace,
332 request,
333 control,
334 )
335 })
336 }
337
338 fn database_id(&self) -> DatabaseId {
339 self.identity
340 }
341 fn private_revision_scope(&self) -> Option<uqa_storage::mvcc::PrivateRevisionScope> {
342 self.native
343 .is_some()
344 .then_some(|key| Ok(native::NativeRecordIdentity::revision_scope(key)))
345 }
346 fn graph_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::GraphRecordLayout> {
347 match self.native.as_ref() {
348 Some(namespace) => Some(namespace),
349 None => Some(&uqa_storage::key_value::KeyValueGraphRecords),
350 }
351 }
352 fn occurrence_record_layout(&self) -> &dyn uqa_storage::mvcc::OccurrenceRecordLayout {
353 if self.native.is_some() {
354 &crate::inverted_index::NativeOccurrenceRecords
355 } else {
356 &uqa_storage::key_value::KeyValueOccurrenceRecords
357 }
358 }
359 fn notification_record_layout(&self) -> &dyn uqa_storage::mvcc::NotificationRecordLayout {
360 match self.native.as_ref() {
361 Some(namespace) => namespace,
362 None => &uqa_storage::key_value::KeyValueNotificationRecords,
363 }
364 }
365
366 fn maintenance_record_layout(&self) -> &dyn uqa_storage::mvcc::MaintenanceRecordLayout {
367 if self.native.is_some() {
368 &native::NativeMaintenanceRecords
369 } else {
370 &uqa_storage::key_value::KeyValueMaintenanceRecords
371 }
372 }
373 fn ivf_record_layout(&self) -> &dyn uqa_storage::mvcc::IVFRecordLayout {
374 if self.native.is_some() {
375 &crate::vector_index::NativeIVFRecords
376 } else {
377 &uqa_storage::key_value::KeyValueIVFRecords
378 }
379 }
380 fn hnsw_record_layout(&self) -> Option<&dyn uqa_storage::mvcc::HNSWRecordLayout> {
381 if self.native.is_some() {
382 Some(&crate::vector_index::NativeHNSWRecords)
383 } else {
384 Some(&uqa_storage::key_value::KeyValueHNSWRecords)
385 }
386 }
387 fn allocate_transaction(
388 &self,
389 control: &StorageReadControl,
390 ) -> VersionResult<StorageTransactionId> {
391 control.cancellation().check()?;
392 self.with_write(control, |connection| {
393 write::allocate(connection, self.identity, self.native, control)
394 })
395 }
396
397 fn allocate_managed_transaction(
398 &self,
399 control: &StorageReadControl,
400 ) -> VersionResult<uqa_storage::mvcc::RetainedTransactionAllocation> {
401 self.allocate_receipt_owner(control)
402 }
403 fn snapshot(
404 &self,
405 control: &StorageReadControl,
406 ) -> VersionResult<Arc<dyn CommittedRecordSnapshot>> {
407 control.cancellation().check()?;
408 let monitor = self.commit_monitor_version()?;
410 let mut reclamation_epoch = 0;
411 let lease = self.snapshots.capture(control, || {
412 self.with(|connection| {
413 let read = connection.unchecked_transaction()?;
414 native::check_mapping(&read, self.native)?;
415 let sequence = codec::header(&read, self.identity)?.sequence;
416 reclamation_epoch = tombstones::epoch(&read)?;
417 read.commit()?;
418 Ok(sequence)
419 })
420 })?;
421 uqa_storage::mvcc::retain_record_snapshot(
422 read::Snapshot {
423 store: self.clone(),
424 sequence: lease.sequence(),
425 reclamation_epoch,
426 monitor: monitor.map(std::sync::atomic::AtomicU64::new),
427 table_owners: read::table_owners::TableOwners::default(),
428 row_presence: read::row_presence::RowPresence::default(),
429 _lease: lease,
430 },
431 control,
432 )
433 }
434 fn commit_monitor_version(&self) -> VersionResult<Option<u64>> {
435 self.connection
436 .commit_monitor_version()
437 .map_err(|error| VersionError::Storage(error.into()))
438 }
439 fn reclaim_versions(&self, control: &StorageReadControl) -> VersionResult<u64> {
440 self.snapshots.reclaim(control, |oldest| {
441 self.with_write(control, |connection| {
442 reclamation::reclaim(connection, self.identity, self.native, oldest, control)
443 })
444 })
445 }
446
447 fn reclaim_tombstones(
448 &self,
449 request: &uqa_storage::mvcc::TombstoneReclamationRequest<'_>,
450 control: &StorageReadControl,
451 ) -> VersionResult<uqa_storage::mvcc::TombstoneReclamationStep> {
452 request.validate(control)?;
453 self.snapshots.reclaim(control, |oldest| {
454 if oldest.is_some() {
455 return Ok(uqa_storage::mvcc::TombstoneReclamationStep::Retained);
456 }
457 self.with_write(control, |connection| {
458 tombstones::reclaim(connection, self.identity, self.native, request, control)
459 })
460 })
461 }
462
463 fn reclaim_diskann_tombstones(&self, control: &StorageReadControl) -> VersionResult<()> {
464 if self.native.is_none() {
465 return uqa_storage::mvcc::reclaim_key_value_diskann_tombstones(self, control);
466 }
467 for family in [
468 native::NativeRecordFamily::DiskANNRecords,
469 native::NativeRecordFamily::VectorOrigins,
470 native::NativeRecordFamily::VectorChanges,
471 native::NativeRecordFamily::VectorPopulations,
472 native::NativeRecordFamily::VectorPopulationWitnesses,
473 ] {
474 let prefix = native::NativeRecordIdentity::family_prefix(family, control)?;
475 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
476 }
477 let prefix = self.vector_field_guard_layout().prefix(control)?;
478 uqa_storage::mvcc::reclaim_tombstone_prefix(self, &prefix, control)?;
479 Ok(())
480 }
481
482 fn vector_field_guard_layout(&self) -> &dyn uqa_storage::mvcc::VectorFieldGuardLayout {
483 self
484 }
485
486 fn commit(
487 &self,
488 transaction: StorageTransactionId,
489 prepared: &PreparedRecordCommit,
490 control: &StorageReadControl,
491 ) -> CommitResult {
492 self.check_transaction(transaction)?;
493 control.cancellation().check().map_err(VersionError::from)?;
494 let _bindings = write::reserve_bindings(prepared, control)?;
495 self.with_write(control, |connection| {
496 let _cache = commit_cache::CommitCache::grow(connection, prepared, control)?;
497 Ok(write::commit(
498 connection,
499 transaction,
500 prepared,
501 self.native,
502 control,
503 ))
504 })?
505 }
506 fn commit_status(
507 &self,
508 transaction: StorageTransactionId,
509 control: &StorageReadControl,
510 ) -> VersionResult<CommitStatus> {
511 self.check_transaction(transaction)?;
512 control.cancellation().check()?;
513 self.with(|connection| {
514 let read = connection.unchecked_transaction()?;
515 native::check_mapping(&read, self.native)?;
516 codec::header(&read, self.identity)?;
517 let status = codec::status(&read, transaction)?;
518 control.cancellation().check().map_err(VersionError::from)?;
519 read.commit()?;
520 Ok(status)
521 })
522 }
523
524 fn acknowledge_transaction(
525 &self,
526 acknowledgement: uqa_storage::mvcc::ReceiptAcknowledgement,
527 control: &StorageReadControl,
528 ) -> VersionResult<()> {
529 self.check_transaction(acknowledgement.transaction())?;
530 self.with_write(control, |connection| {
531 receipts::acknowledge(connection, self.native, acknowledgement, control)
532 })
533 }
534
535 fn reclaim_transaction_receipts(&self, control: &StorageReadControl) -> VersionResult<u64> {
536 self.reclaim_receipts(control)
537 }
538 fn abort(
539 &self,
540 transaction: StorageTransactionId,
541 control: &StorageReadControl,
542 ) -> VersionResult<CommitStatus> {
543 self.check_transaction(transaction)?;
544 control.cancellation().check()?;
545 self.with(|connection| write::abort(connection, transaction, self.native, control))
546 }
547}